diff --git a/android/app/build.gradle b/android/app/build.gradle index 0991728a..f3245d6a 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -11,8 +11,8 @@ android { applicationId "com.compassconnections.app" minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion - versionCode 131 - versionName "1.31.0" + versionCode 132 + versionName "1.32.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" aaptOptions { // Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps. diff --git a/backend/api/src/app.ts b/backend/api/src/app.ts index b2a53d7d..6e28c517 100644 --- a/backend/api/src/app.ts +++ b/backend/api/src/app.ts @@ -92,6 +92,7 @@ import {setLastOnlineTime} from './set-last-online-time' import {shipProfiles} from './ship-profiles' import {starProfile} from './star-profile' import {stats} from './stats' +import {transcribeAudio} from './transcribe-audio' import {unsubscribe} from './unsubscribe' import {updateEvent} from './update-event' import {updateMe} from './update-me' @@ -653,6 +654,7 @@ const handlers: {[k in APIPath]: APIHandler} = { vote: vote, 'validate-username': validateUsernameEndpoint, 'llm-extract-profile': llmExtractProfileEndpoint, + 'transcribe-audio': transcribeAudio, // 'user/:username': getUser, // 'user/:username/lite': getDisplayUser, // 'user/by-id/:id/lite': getDisplayUser, @@ -678,7 +680,8 @@ Object.entries(handlers).forEach(([path, handler]) => { const apiRoute = [ url, - express.json({limit: '1mb'}), + // Endpoints that carry binary payloads (e.g. base64 audio) declare a larger `bodyLimit`. + express.json({limit: (api as any).bodyLimit ?? '1mb'}), allowCorsUnrestricted, cache, typedEndpoint(path as any, handler as any), diff --git a/backend/api/src/llm-extract-profile.ts b/backend/api/src/llm-extract-profile.ts index 45c06149..d442f3f2 100644 --- a/backend/api/src/llm-extract-profile.ts +++ b/backend/api/src/llm-extract-profile.ts @@ -9,6 +9,8 @@ import { GENDERS, LANGUAGE_CHOICES, MBTI_CHOICES, + NEUROTYPE_CHOICES, + ORIENTATION_CHOICES, POLITICAL_CHOICES, PSYCHEDELICS_CHOICES, RACE_CHOICES, @@ -23,7 +25,7 @@ import {debug} from 'common/logger' import {ProfileWithoutUser} from 'common/profiles/profile' import {SITE_ORDER} from 'common/socials' import {removeNullOrUndefinedProps} from 'common/util/object' -import {parseJsonContentToText} from 'common/util/parse' +import {parseJsonContentToText, textToJSONContent} from 'common/util/parse' import {HOUR_MS, MINUTE_MS, sleep} from 'common/util/time' import {createHash} from 'crypto' import {promises as fs} from 'fs' @@ -38,18 +40,26 @@ const CACHE_DIR = join(tmpdir(), 'compass-llm-cache') const CACHE_TTL_MS = 24 * HOUR_MS const PROCESSING_TTL_MS = 10 * MINUTE_MS +type ExtractSource = 'text' | 'url' | 'voice' + interface ParsedBody { content?: string url?: string locale?: string + source?: ExtractSource } +// Bump whenever the extraction prompt changes. The cache key is otherwise derived purely from the +// request, so a prompt fix would keep returning the old answer for the 24h TTL — which looks exactly +// like the fix not working. +const PROMPT_VERSION = 2 + function getCacheKey(parsedBody: ParsedBody): string { if (!USE_CACHE) return '' const hash = createHash('sha256') // Normalize: sort keys for consistent hashing const normalized = JSON.stringify(parsedBody, Object.keys(parsedBody).sort()) - hash.update(normalized) + hash.update(`v${PROMPT_VERSION}:${normalized}`) return hash.digest('hex') } @@ -79,6 +89,8 @@ async function validateProfileFields( 'cannabis_intention', 'psychedelics_pref', 'cannabis_pref', + 'orientation', + 'neurotype', ] for (const key of toArray) { if (result[key] !== undefined) { @@ -113,6 +125,10 @@ async function validateProfileFields( 'occupation_title', 'religious_beliefs', 'political_details', + 'gender_details', + 'orientation_details', + 'neurotype_details', + 'accessibility_notes', ] for (const key of toString) { if (result[key] !== undefined) { @@ -246,7 +262,10 @@ async function setCachedResult(cacheKey: string, result: any): Promise { await fs.mkdir(CACHE_DIR, {recursive: true}) const cacheFile = join(CACHE_DIR, `${cacheKey}.json`) await fs.writeFile(cacheFile, JSON.stringify(result), 'utf-8') - debug('Cached LLM result', {cacheKey: cacheKey.substring(0, 8), result}) + debug('Cached LLM result', { + cacheKey: cacheKey.substring(0, 8), + result: JSON.stringify(result), + }) } catch (error) { log('Failed to write cache', {cacheKey, error}) // Don't throw - caching failure shouldn't break the main flow @@ -296,11 +315,13 @@ async function processAndCache( content?: string | undefined, url?: string | undefined, locale?: string, + source?: ExtractSource, ): Promise { log('Extracting profile from content', { contentLength: content?.length, url, locale, + source, }) try { let bio: JSONContent | undefined @@ -309,7 +330,7 @@ async function processAndCache( debug(JSON.stringify(bio, null, 2)) content = parseJsonContentToText(bio) } - const profile = await callLLM(content, locale) + const profile = await callLLM(content, locale, source) if (bio) { profile.bio = bio } @@ -422,7 +443,9 @@ async function _callClaude(text: string) { export async function callLLM( content: string, locale?: string, + source?: ExtractSource, ): Promise> { + const isVoice = source === 'voice' const [INTERESTS, CAUSE_AREAS, WORK_AREAS] = await Promise.all([ getOptions('interests', locale), getOptions('causes', locale), @@ -451,12 +474,19 @@ export async function callLLM( cannabis_intention: Object.values(SUBSTANCE_INTENTION_CHOICES), psychedelics_pref: Object.values(SUBSTANCE_PREFERENCE_CHOICES), cannabis_pref: Object.values(SUBSTANCE_PREFERENCE_CHOICES), + orientation: Object.values(ORIENTATION_CHOICES), + neurotype: Object.values(NEUROTYPE_CHOICES), } const PROFILE_FIELDS: Partial> = { // Basic info age: 'Number. Age in years (between 18 and 100).', gender: `String. One of: ${validChoices.pref_gender?.join(', ')}. If multiple mentioned, use the most likely one. Infer if you have enough evidence`, + gender_details: + 'String. Free-form elaboration on their gender identity, only if they say more than the label itself.', + orientation: `Array. Any of: ${validChoices.orientation?.join(', ')}. Only if stated — never infer from the gender of a partner or of who they are looking for.`, + orientation_details: + 'String. Free-form elaboration on their orientation, only if they say more than the label itself.', height_in_inches: 'Number. Height converted to inches.', city: 'String. Current city of residence (English spelling).', country: 'String. Current country of residence (English spelling).', @@ -498,6 +528,11 @@ export async function callLLM( big5_agreeableness: 'Number 0–100. Only if explicitly self-reported, never infer.', big5_neuroticism: 'Number 0–100. Only if explicitly self-reported, never infer.', + // Neurotype is an identity here, not a diagnosis: only ever take the person's own words for it. + neurotype: `Array. Any of: ${validChoices.neurotype?.join(', ')}. Only if they identify this way themselves — never infer it from how they describe their personality, focus, energy or social life.`, + neurotype_details: + 'String. Free-form elaboration on their neurotype, only if they say more than the label itself.', + // Beliefs religion: `Array. Any of: ${validChoices.religion?.join(', ')}`, religious_beliefs: @@ -511,7 +546,7 @@ export async function callLLM( 'Number. Minimum preferred age of match (higher than 18, only if mentioned, do NOT infer).', pref_age_max: 'Number. Maximum preferred age of match (lower than 100, only if mentioned, do NOT infer).', - pref_gender: `Array. Any of: ${validChoices.pref_gender?.join(', ')}`, + pref_gender: `Array. Any of: ${validChoices.pref_gender?.join(', ')}. Only the genders they actually name as sought. If they say gender does not matter to them, or is unimportant to their attraction, OMIT this field — an omitted field already means "no preference", so listing every option instead is both wrong and unreadable.`, pref_relation_styles: `Array. Any of: ${validChoices.pref_relation_styles?.join(', ')}`, pref_romantic_styles: `Array. Any of: ${validChoices.pref_romantic_styles?.join(', ')}`, relationship_status: `Array. Any of: ${validChoices.relationship_status?.join(', ')}`, @@ -523,6 +558,8 @@ export async function callLLM( headline: 'String. Summary of who they are, in their own voice (first person). Maximum 200 characters total. Cannot be null.', keywords: 'Array of 3–6 short tags summarising the person.', + accessibility_notes: + 'String. Practical things that help someone meet them well — access needs, energy levels, sensory preferences, venue preferences. Only if mentioned; never infer a disability, and keep their own framing and wording.', links: `Object. Key is any of: ${SITE_ORDER.join(', ')}.`, // Taxonomies — match existing labels first, only add new if truly no close match exists @@ -531,7 +568,25 @@ export async function callLLM( work: `Array. Use only existing labels, do not add new if no close match. Any of: ${validChoices.work?.join(', ')}`, } - const EXTRACTION_PROMPT = `You are a profile information extraction expert analyzing text from a personal webpage, bio, or similar source. + // For text and URL sources the bio is the source material itself, stored verbatim. A speech + // transcript makes a poor bio (filler words, false starts, no paragraphs), so for voice we ask + // the model to write it instead. + if (isVoice) { + PROFILE_FIELDS.bio = + 'String. A first-person bio written from what the person said, in their own voice and their ' + + 'own language. Keep their wording and personality wherever you can; only clean up filler ' + + 'words, false starts, repetitions and transcription noise, and organise it into ' + + 'paragraphs. Separate each paragraph from the next with a BLANK LINE, i.e. two newline ' + + 'characters ("\\n\\n") — a single newline is not enough. Never add facts, opinions or ' + + 'flourishes they did not say, and never write about them in the third person. Plain text ' + + 'only — no markdown.' + } + + const EXTRACTION_PROMPT = `You are a profile information extraction expert analyzing ${ + isVoice + ? 'a speech-to-text transcript of someone talking about themselves out loud' + : 'text from a personal webpage, bio, or similar source' + }. TASK: Extract structured profile data and return it as a single valid JSON object. @@ -540,12 +595,20 @@ RULES: - Omit the key in the output for missing fields - For taxonomy fields (interests, causes, work): match existing labels first; only add a new label if truly no existing one is close - For big5 scores: only populate if the person explicitly states a test result — never infer from personality description -- Return valid JSON only — no markdown, no explanation, no extra text +- Never answer a multi-choice field by selecting every option it offers. An expression of openness or indifference ("gender doesn't matter to me", "I'm open to anything") is not a selection of all values — omit the field, which already means "no preference" +- Return valid JSON only — no markdown, no explanation, no extra text${ + isVoice + ? ` +- The transcript is spoken language: expect filler words, false starts, self-corrections and speech-recognition errors. Read past them, and when the person corrects themselves keep the corrected version +- Ignore anything the person says to the recorder rather than about themselves (e.g. "let me start over", "what else should I say") +- Spoken numbers, places and names may be mis-transcribed; only fill a field when you are confident what was meant` + : '' + } SCHEMA (each value describes the expected type and accepted values): ${JSON.stringify(PROFILE_FIELDS, null, 2)} -TEXT TO ANALYZE: +${isVoice ? 'TRANSCRIPT TO ANALYZE' : 'TEXT TO ANALYZE'}: ` const text = EXTRACTION_PROMPT + content if (text.length > MAX_CONTEXT_LENGTH) { @@ -564,6 +627,10 @@ TEXT TO ANALYZE: try { parsed = typeof outputText === 'string' ? JSON.parse(outputText) : outputText parsed = await validateProfileFields(parsed, validChoices) + // The bio column holds rich text; the model answers with plain prose. + if (typeof parsed.bio === 'string') { + parsed.bio = parsed.bio.trim() ? textToJSONContent(parsed.bio) : undefined + } parsed = removeNullOrUndefinedProps(parsed) } catch (parseError) { log('Failed to parse LLM response as JSON', {outputText, parseError}) @@ -647,7 +714,7 @@ export async function fetchOnlineProfile(url: string | undefined): Promise = async (parsedBody) => { - const {url, locale} = parsedBody + const {url, locale, source} = parsedBody const content = parsedBody.content if (content && url) { @@ -672,7 +739,7 @@ export const llmExtractProfileEndpoint: APIHandler<'llm-extract-profile'> = asyn await setProcessing(cacheKey) // Kick off async processing (don't await) - processAndCache(cacheKey, content, url, locale).catch((err) => { + processAndCache(cacheKey, content, url, locale, source).catch((err) => { log('Unexpected error in async processing', {cacheKey, error: err}) }) diff --git a/backend/api/src/transcribe-audio.ts b/backend/api/src/transcribe-audio.ts new file mode 100644 index 00000000..db8bd22d --- /dev/null +++ b/backend/api/src/transcribe-audio.ts @@ -0,0 +1,100 @@ +import {APIErrors, APIHandler} from 'api/helpers/endpoint' +import {debug} from 'common/logger' +import {log} from 'shared/monitoring/log' + +// Whisper's own hard limit is 25 MB. We cap below that so we reject with a friendly message rather +// than letting OpenAI 413 on us. Opus runs ~200-400 KB/minute, so this is many minutes of speech. +const MAX_AUDIO_BYTES = 20 * 1024 * 1024 + +// `gpt-4o-transcribe` is Whisper's successor and the default in our other voice codebase; set +// OPENAI_TRANSCRIBE_MODEL=whisper-1 to fall back to classic Whisper. +const TRANSCRIBE_MODEL = process.env.OPENAI_TRANSCRIBE_MODEL || 'gpt-4o-transcribe' + +// Container → file extension. OpenAI infers the codec from the uploaded filename, so this has to be +// right; `;codecs=...` suffixes that browsers append are stripped first. +const EXTENSION_BY_MIME: Record = { + 'audio/webm': 'webm', + 'audio/ogg': 'ogg', + 'audio/mp4': 'mp4', + 'audio/m4a': 'm4a', + 'audio/x-m4a': 'm4a', + 'audio/mpeg': 'mp3', + 'audio/mp3': 'mp3', + 'audio/wav': 'wav', + 'audio/x-wav': 'wav', + 'audio/flac': 'flac', +} + +function parseMimeType(mimeType: string): {contentType: string; extension: string} { + const base = mimeType.split(';')[0].trim().toLowerCase() + const extension = EXTENSION_BY_MIME[base] + if (extension) return {contentType: base, extension} + // Unknown container: webm/opus is what every Chromium browser records, so it is the safest guess. + log('Unsupported audio mime type, falling back to audio/webm', {mimeType}) + return {contentType: 'audio/webm', extension: 'webm'} +} + +// Whisper takes an ISO-639-1 code; our locales are either 'en' or 'en-US'-shaped. +function toLanguageCode(locale: string | undefined): string | undefined { + const code = locale?.split(/[-_]/)[0]?.toLowerCase() + return code && /^[a-z]{2}$/.test(code) ? code : undefined +} + +export const transcribeAudio: APIHandler<'transcribe-audio'> = async (props) => { + const {audio, mimeType, locale} = props + + const apiKey = process.env.OPENAI_API_KEY + if (!apiKey) { + log('OPENAI_API_KEY not configured') + throw APIErrors.internalServerError('Voice transcription is not configured') + } + + // `audio` is base64: 4 characters encode 3 bytes. + const sizeInBytes = Math.floor((audio.length * 3) / 4) + if (sizeInBytes > MAX_AUDIO_BYTES) { + throw APIErrors.badRequest('Recording is too long. Please record a shorter message.') + } + + const {contentType, extension} = parseMimeType(mimeType) + const language = toLanguageCode(locale) + log('Transcribing audio', {mimeType, contentType, sizeInBytes, model: TRANSCRIBE_MODEL, language}) + + const form = new FormData() + form.append( + 'file', + new Blob([Buffer.from(audio, 'base64')], {type: contentType}), + `voice.${extension}`, + ) + form.append('model', TRANSCRIBE_MODEL) + // Punctuated, readable text — the extraction step downstream reads much better prose than a + // single unbroken run of words. + form.append('response_format', 'text') + if (language) form.append('language', language) + + let response: Response + try { + response = await fetch('https://api.openai.com/v1/audio/transcriptions', { + method: 'POST', + headers: {Authorization: `Bearer ${apiKey}`}, + body: form, + }) + } catch (error) { + log('OpenAI transcription request failed', {error}) + throw APIErrors.internalServerError('Failed to transcribe the recording') + } + + if (!response.ok) { + const errorText = await response.text() + log('OpenAI transcription API error', {status: response.status, error: errorText}) + throw APIErrors.internalServerError('Failed to transcribe the recording') + } + + const transcript = (await response.text()).trim() + debug({transcript}) + + if (!transcript) { + throw APIErrors.badRequest('We could not hear any speech in that recording. Please try again.') + } + + return {transcript} +} diff --git a/common/messages/de.json b/common/messages/de.json index b3fc3789..5e1a20b2 100644 --- a/common/messages/de.json +++ b/common/messages/de.json @@ -1050,6 +1050,35 @@ "profile.llm.extract.description": "Füllen Sie Ihr Profil automatisch aus, indem Sie einen Link (Notion, Google Docs, persönliche Website usw.) einfügen oder Ihren Inhalt direkt einfügen.", "profile.llm.extract.guidance": "Achtung: Wir verwenden Google AI, um Ihre Informationen zu extrahieren. Google kann diesen Inhalt verwenden, um seine Modelle zu verbessern. Bevorzugen Sie Privatsphäre? Füllen Sie das Formular einfach manuell aus — keine KI beteiligt.", "profile.llm.extract.placeholder": "Fügen Sie eine URL ein oder fügen Sie hier Ihren Profilinhalt ein.", + "profile.autofill.by_voice": "Per Sprache", + "profile.autofill.by_link": "Per Link oder Text", + "profile.voice.description": "Sprechen Sie lieber, als zu schreiben? Nehmen Sie sich ein bis zwei Minuten auf, und wir füllen Ihr Profil aus dem aus, was Sie sagen.", + "profile.voice.guidance": "Achtung: Ihre Aufnahme wird zur Transkription an OpenAI Whisper geschickt und das Transkript an Google AI, um die Felder auszufüllen. Wir zahlen für beide Dienste, daher sollte keiner Ihre Inhalte zum Trainieren seiner Modelle verwenden — aber wissen kann man es nie. Bevorzugen Sie Privatsphäre? Füllen Sie das Formular einfach manuell aus — keine KI beteiligt.", + "profile.voice.prompts.title": "Sprechen Sie frei, oder nehmen Sie diese Fragen als Ausgangspunkt:", + "profile.voice.prompt.who": "Wer sind Sie, und wo leben Sie?", + "profile.voice.prompt.work": "Was machen Sie — Arbeit, Studium, Projekte?", + "profile.voice.prompt.time": "Wie verbringen Sie Ihre Zeit am liebsten?", + "profile.voice.prompt.care": "Was ist Ihnen am wichtigsten?", + "profile.voice.prompt.looking": "Welche Art von Verbindung suchen Sie?", + "profile.voice.prompts.footer": "Es muss nicht perfekt sein — wir räumen Ähs und Fehlstarts auf. Lassen Sie weg, was Sie nicht teilen möchten.", + "profile.voice.tap_to_record": "Zum Aufnehmen tippen", + "profile.voice.start": "Aufnahme starten", + "profile.voice.stop": "Aufnahme beenden", + "profile.voice.recording": "Aufnahme läuft…", + "profile.voice.paused": "Pausiert", + "profile.voice.pause": "Pausieren", + "profile.voice.resume": "Fortsetzen", + "profile.voice.done_talking": "Ich bin fertig", + "profile.voice.review_recording": "Hören Sie es sich vor der Transkription an — {duration} aufgenommen.", + "profile.voice.transcribe": "Meine Aufnahme transkribieren", + "profile.voice.transcribing": "Transkribiere…", + "profile.voice.record_again": "Neu aufnehmen", + "profile.voice.review_transcript": "Das haben wir gehört. Sie müssen nichts aufräumen: Ähs, Fehlstarts und die Stellen, an denen Sie sich mitten im Satz korrigiert haben, übernehmen wir für Sie — nichts davon landet in Ihrem Profil. Zu korrigieren lohnt sich nur, was wir falsch verstanden haben: Namen und Orte sind die üblichen Verdächtigen.", + "profile.voice.fill_profile": "Mein Profil ausfüllen", + "profile.voice.error.permission": "Wir konnten nicht auf Ihr Mikrofon zugreifen. Erlauben Sie den Mikrofonzugriff in Ihrem Browser, und versuchen Sie es erneut.", + "profile.voice.error.unsupported": "Ihr Browser unterstützt keine Sprachaufnahme. Nutzen Sie stattdessen die Option Link oder Text.", + "profile.voice.error.failed": "Die Aufnahme ist fehlgeschlagen. Bitte versuchen Sie es erneut.", + "profile.voice.error.transcription": "Wir konnten Ihre Aufnahme nicht transkribieren. Sie ist noch da — versuchen Sie es erneut, oder füllen Sie Ihr Profil manuell aus.", "profile.mbti": "MBTI", "profile.mbti.INTJ": "Architekt", "profile.mbti.INTP": "Logiker", @@ -1130,6 +1159,8 @@ "profile.optional.og_card": "Profilkarte", "profile.optional.orientation": "Sexuelle Orientierung", "profile.optional.photos": "Fotos", + "profile.optional.photo_reminder": "Eines können wir nicht für Sie ausfüllen: Ihre Fotos. Sie sind das Erste, was die Leute ansehen — gehen Sie also nicht, ohne ein paar hinzuzufügen.", + "profile.optional.photo_reminder.cta": "Zu den Fotos", "profile.optional.political_beliefs": "Politische Ansichten", "profile.optional.raised_in": "Ort, an dem ich aufgewachsen bin", "profile.optional.raised_in_hint": "Besonders nützlich, wenn Sie in einem anderen Land aufgewachsen sind als dem, in dem Sie jetzt leben – und wenn dies Ihre kulturellen Referenzen, Werte und Lebenserfahrungen widerspiegelt.", diff --git a/common/messages/fr.json b/common/messages/fr.json index 4749f9fe..7d90780a 100644 --- a/common/messages/fr.json +++ b/common/messages/fr.json @@ -1049,6 +1049,35 @@ "profile.llm.extract.description": "Remplissez automatiquement votre profil en déposant un lien (Notion, Google Docs, site personnel, etc.) ou en collant directement votre contenu.", "profile.llm.extract.guidance": "Attention : nous utilisons Google AI pour extraire vos informations. Comme nous payons pour leur service, Google ne devrait pas utiliser ce contenu pour améliorer ses modèles — mais on ne sait jamais. Vous préférez garder les choses privées ? Remplissez simplement le formulaire manuellement — aucune IA impliquée.", "profile.llm.extract.placeholder": "Insérez une URL ou collez le contenu de votre profil ici.", + "profile.autofill.by_voice": "À la voix", + "profile.autofill.by_link": "Par lien ou texte", + "profile.voice.description": "Vous préférez parler plutôt qu'écrire ? Enregistrez-vous pendant une ou deux minutes et nous remplirons votre profil à partir de ce que vous dites.", + "profile.voice.guidance": "Attention : votre enregistrement est envoyé à OpenAI Whisper pour être transcrit, et la transcription à Google AI pour remplir les champs. Comme nous payons pour ces services, ni l'un ni l'autre ne devrait utiliser votre contenu pour entraîner ses modèles — mais on ne sait jamais. Vous préférez garder les choses privées ? Remplissez simplement le formulaire manuellement — aucune IA impliquée.", + "profile.voice.prompts.title": "Parlez librement, ou partez de ces questions :", + "profile.voice.prompt.who": "Qui êtes-vous, et où vivez-vous ?", + "profile.voice.prompt.work": "Que faites-vous — travail, études, projets ?", + "profile.voice.prompt.time": "Comment aimez-vous passer votre temps ?", + "profile.voice.prompt.care": "Qu'est-ce qui vous tient le plus à cœur ?", + "profile.voice.prompt.looking": "Quel type de connexion recherchez-vous ?", + "profile.voice.prompts.footer": "Pas besoin d'être parfait — nous nettoyons les hésitations et les faux départs. Passez ce que vous préférez ne pas partager.", + "profile.voice.tap_to_record": "Appuyez pour enregistrer", + "profile.voice.start": "Démarrer l'enregistrement", + "profile.voice.stop": "Arrêter l'enregistrement", + "profile.voice.recording": "Enregistrement…", + "profile.voice.paused": "En pause", + "profile.voice.pause": "Mettre en pause", + "profile.voice.resume": "Reprendre", + "profile.voice.done_talking": "J'ai terminé", + "profile.voice.review_recording": "Écoutez-vous avant la transcription — {duration} enregistré.", + "profile.voice.transcribe": "Transcrire mon enregistrement", + "profile.voice.transcribing": "Transcription…", + "profile.voice.record_again": "Réenregistrer", + "profile.voice.review_transcript": "Voici ce que nous avons entendu. Pas besoin de le nettoyer : les hésitations, les faux départs, les fois où vous avez changé d'avis en cours de phrase sont gérés pour vous, et rien de tout cela ne se retrouvera sur votre profil. La seule chose à corriger, c'est ce que nous avons mal entendu : les noms et les lieux sont les suspects habituels.", + "profile.voice.fill_profile": "Remplir mon profil", + "profile.voice.error.permission": "Nous n'avons pas pu accéder à votre microphone. Autorisez l'accès au microphone dans votre navigateur, puis réessayez.", + "profile.voice.error.unsupported": "Votre navigateur ne prend pas en charge l'enregistrement vocal. Essayez plutôt l'option lien ou texte.", + "profile.voice.error.failed": "L'enregistrement a échoué. Veuillez réessayer.", + "profile.voice.error.transcription": "Nous n'avons pas pu transcrire votre enregistrement. Il est toujours là — réessayez, ou remplissez votre profil manuellement.", "profile.mbti": "MBTI", "profile.mbti.INTJ": "Architecte", "profile.mbti.INTP": "Logicien", @@ -1129,6 +1158,8 @@ "profile.optional.og_card": "Carte de profil", "profile.optional.orientation": "Orientation sexuelle", "profile.optional.photos": "Photos", + "profile.optional.photo_reminder": "Une chose que nous ne pouvons pas remplir à votre place : vos photos. C'est la première chose que les gens regardent, alors ne partez pas sans en ajouter quelques-unes.", + "profile.optional.photo_reminder.cta": "Aller aux photos", "profile.optional.political_beliefs": "Opinions politiques", "profile.optional.raised_in": "Lieu où j'ai grandi", "profile.optional.raised_in_hint": "Particulièrement utile si vous avez grandi dans un pays différent de celui où vous vivez maintenant – et si cela reflète vos références culturelles, vos valeurs et vos expériences de vie.", diff --git a/common/src/api/schema.ts b/common/src/api/schema.ts index f6172888..1b09efdf 100644 --- a/common/src/api/schema.ts +++ b/common/src/api/schema.ts @@ -55,6 +55,13 @@ export type APIGenericSchema = { */ rateLimited?: boolean + /** + * Maximum request body size, as an `express.json({limit})` value (e.g. '20mb'). + * Only needed for endpoints carrying binary payloads such as base64-encoded audio. + * @default '1mb' + */ + bodyLimit?: string + /** * Zod schema for request validation * - For GET requests: Validates query parameters @@ -1285,6 +1292,10 @@ export const API = (_apiTypeCheck = { content: z.string().min(1).optional(), url: z.string().url().optional(), locale: z.string().optional(), + // Where `content` came from. 'voice' means it is a speech transcript, so the LLM has to be + // told to read past filler words and to write the bio itself rather than us storing the + // transcript verbatim. + source: z.enum(['text', 'url', 'voice']).optional(), }) .strict(), returns: {} as { @@ -1294,6 +1305,27 @@ export const API = (_apiTypeCheck = { summary: 'Extract profile information from text using LLM', tag: 'Profiles', }, + 'transcribe-audio': { + method: 'POST', + authed: true, + rateLimited: true, + // Audio arrives base64-encoded in the JSON body, which is far bigger than the default 1mb. + bodyLimit: '20mb', + props: z + .object({ + // Base64-encoded audio (no data: URI prefix). + audio: z.string().min(1), + // Recording container/codec as reported by MediaRecorder, e.g. 'audio/webm;codecs=opus'. + mimeType: z.string().min(1), + locale: z.string().optional(), + }) + .strict(), + returns: {} as { + transcript: string + }, + summary: 'Transcribe a voice recording to text', + tag: 'Profiles', + }, 'get-user-journeys': { method: 'GET', authed: true, diff --git a/common/src/secrets.ts b/common/src/secrets.ts index 7514a114..aba1b1c7 100644 --- a/common/src/secrets.ts +++ b/common/src/secrets.ts @@ -30,6 +30,7 @@ export const secrets = ( 'DB_ENC_MASTER_KEY_BASE64', 'GOOGLE_CLIENT_SECRET', 'GEMINI_API_KEY', + 'OPENAI_API_KEY', // Some typescript voodoo to keep the string literal types while being not readonly. ] as const ).concat() diff --git a/common/src/util/parse.ts b/common/src/util/parse.ts index d33c854f..7b7f7165 100644 --- a/common/src/util/parse.ts +++ b/common/src/util/parse.ts @@ -83,6 +83,61 @@ export function parseJsonContentToText(content: JSONContent | string | undefined return typeof content === 'string' ? content : richTextToString(content) } +/** + * Inverse of `parseJsonContentToText`: turns plain text into a minimal Tiptap document. Used when + * an LLM hands us prose (e.g. a bio written from a voice transcript) that has to be stored in a + * rich-text column. + * + * A blank line always starts a new paragraph, and single newlines inside such a block are hard + * breaks. But when the text contains no blank line at all, single newlines are the only structure + * on offer, so they are read as paragraph breaks instead — LLMs routinely separate paragraphs with + * one `\n` however firmly the prompt asks for two, and the alternative is one giant paragraph. + */ +export function textToJSONContent(text: string): JSONContent { + const separator = /\n\s*\n/.test(text) ? /\n\s*\n+/ : '\n' + const paragraphs = text + .split(separator) + .map((block) => block.trim()) + .filter((block) => block.length > 0) + + return { + type: 'doc', + content: paragraphs.length + ? paragraphs.map((block) => ({ + type: 'paragraph', + // Single newlines inside a block are line breaks, not paragraph breaks. + content: block + .split('\n') + .flatMap((line, i) => [ + ...(i > 0 ? [{type: 'hardBreak'}] : []), + ...(line.trim() ? [{type: 'text', text: line.trim()}] : []), + ]), + })) + : [{type: 'paragraph'}], + } +} + +/** + * Appends one rich-text document to another, so a second pass adds to what is already written + * instead of replacing it (e.g. recording a follow-up voice note to extend an existing bio). + * + * Each side is trimmed of its leading and trailing empty paragraphs first, so the join does not + * accumulate blank space every time something is appended. + */ +export function concatJSONContent( + first: JSONContent | null | undefined, + second: JSONContent | null | undefined, +): JSONContent { + const blocksOf = (doc: JSONContent | null | undefined) => { + if (!doc) return [] + const cleaned = cleanDoc(doc) + return Array.isArray(cleaned.content) ? cleaned.content : [] + } + + const content = [...blocksOf(first), ...blocksOf(second)] + return {type: 'doc', content: content.length ? content : [{type: 'paragraph'}]} +} + export function urlBase64ToUint8Array(base64String: string) { const padding = '='.repeat((4 - (base64String.length % 4)) % 4) const base64 = (base64String + padding).replace(/-/g, '+').replace(/_/g, '/') diff --git a/common/tests/unit/textToJSONContent.test.ts b/common/tests/unit/textToJSONContent.test.ts new file mode 100644 index 00000000..7c0d4352 --- /dev/null +++ b/common/tests/unit/textToJSONContent.test.ts @@ -0,0 +1,85 @@ +import {concatJSONContent, parseJsonContentToText, textToJSONContent} from 'common/util/parse' + +describe('textToJSONContent', () => { + it('makes one paragraph per blank-line-separated block', () => { + expect(textToJSONContent('First para.\n\nSecond para.')).toEqual({ + type: 'doc', + content: [ + {type: 'paragraph', content: [{type: 'text', text: 'First para.'}]}, + {type: 'paragraph', content: [{type: 'text', text: 'Second para.'}]}, + ], + }) + }) + + // The LLM writing our bios routinely uses one '\n' between paragraphs no matter how firmly the + // prompt asks for two; without this the whole bio collapsed into a single paragraph. + it('treats single newlines as paragraph breaks when there is no blank line anywhere', () => { + expect(textToJSONContent('One\nTwo')).toEqual({ + type: 'doc', + content: [ + {type: 'paragraph', content: [{type: 'text', text: 'One'}]}, + {type: 'paragraph', content: [{type: 'text', text: 'Two'}]}, + ], + }) + }) + + it('keeps single newlines as hard breaks once blank lines establish the paragraphing', () => { + expect(textToJSONContent('One\nTwo\n\nThree')).toEqual({ + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{type: 'text', text: 'One'}, {type: 'hardBreak'}, {type: 'text', text: 'Two'}], + }, + {type: 'paragraph', content: [{type: 'text', text: 'Three'}]}, + ], + }) + }) + + it('collapses runs of blank lines and trims whitespace', () => { + expect(textToJSONContent(' A \n\n \n\n B ')).toEqual({ + type: 'doc', + content: [ + {type: 'paragraph', content: [{type: 'text', text: 'A'}]}, + {type: 'paragraph', content: [{type: 'text', text: 'B'}]}, + ], + }) + }) + + it('returns a valid empty doc for empty input', () => { + expect(textToJSONContent(' ')).toEqual({type: 'doc', content: [{type: 'paragraph'}]}) + }) + + it('round-trips through the rich text serializer', () => { + const text = 'I grew up in Liège.\n\nI now live in Brussels.' + expect(parseJsonContentToText(textToJSONContent(text))).toBe(text) + }) +}) + +describe('concatJSONContent', () => { + const doc = (...texts: string[]) => ({ + type: 'doc', + content: texts.map((text) => ({type: 'paragraph', content: [{type: 'text', text}]})), + }) + + it('appends the second document after the first', () => { + expect(concatJSONContent(doc('One'), doc('Two'))).toEqual(doc('One', 'Two')) + }) + + it('does not accumulate blank paragraphs at the join', () => { + const trailingBlank = { + type: 'doc', + content: [{type: 'paragraph', content: [{type: 'text', text: 'One'}]}, {type: 'paragraph'}], + } + expect(concatJSONContent(trailingBlank, doc('Two'))).toEqual(doc('One', 'Two')) + }) + + it('returns the other side when one is missing or empty', () => { + expect(concatJSONContent(null, doc('Only'))).toEqual(doc('Only')) + expect(concatJSONContent(doc('Only'), undefined)).toEqual(doc('Only')) + }) + + it('returns a valid empty doc when both sides are empty', () => { + expect(concatJSONContent(null, null)).toEqual({type: 'doc', content: [{type: 'paragraph'}]}) + }) +}) diff --git a/web/components/bio/editable-bio.tsx b/web/components/bio/editable-bio.tsx index 2e83e97b..63ba2510 100644 --- a/web/components/bio/editable-bio.tsx +++ b/web/components/bio/editable-bio.tsx @@ -18,6 +18,13 @@ import {updateProfile} from 'web/lib/api' import {useT} from 'web/lib/locale' import {track} from 'web/lib/service/analytics' +/** + * Restores paragraph spacing for bios. The shared editor styling zeroes `p` margins at this size, + * which suits single-paragraph comments and chat bubbles but runs a multi-paragraph bio together. + * Must be `!important`: it competes with `prose-p:my-0` on the very same element. + */ +export const BIO_PARAGRAPH_SPACING = 'prose-p:!my-3' + export function BioTips({onClick}: {onClick?: () => void}) { const t = useT() const tips = t( @@ -170,6 +177,10 @@ export function BaseBio({defaultValue, onBlur, onEditor, onClickTips}: BaseBioPr // extensions: [StarterKit], max: MAX_DESCRIPTION_LENGTH, defaultValue: defaultValue, + // `proseClass` zeroes paragraph margins at this size — right for comments and chat bubbles, + // which are usually a single paragraph, but it makes a multi-paragraph bio render as one + // undifferentiated wall of text. `!` because it has to beat `prose-p:my-0` on the same element. + className: BIO_PARAGRAPH_SPACING, placeholder: t( 'profile.bio.placeholder', "Tell us all the details about yourself — and what you're looking for!", diff --git a/web/components/bio/profile-bio-block.tsx b/web/components/bio/profile-bio-block.tsx index 5760d71d..7bec4165 100644 --- a/web/components/bio/profile-bio-block.tsx +++ b/web/components/bio/profile-bio-block.tsx @@ -11,7 +11,7 @@ import {Tooltip} from 'web/components/widgets/tooltip' import {updateProfile} from 'web/lib/api' import {useT} from 'web/lib/locale' -import {EditableBio} from './editable-bio' +import {BIO_PARAGRAPH_SPACING, EditableBio} from './editable-bio' export function BioBlock(props: { isCurrentUser: boolean @@ -33,7 +33,10 @@ export function BioBlock(props: { {!edit && profile.bio && ( - + )} {edit && ( diff --git a/web/components/llm-extract-section.tsx b/web/components/llm-extract-section.tsx index 3a8d5718..95a69a19 100644 --- a/web/components/llm-extract-section.tsx +++ b/web/components/llm-extract-section.tsx @@ -45,7 +45,7 @@ export function LLMExtractSection({
{t( 'profile.llm.extract.guidance', - 'Heads up: we use Google AI to extract your info. As we pay for the service, Google should not use this content to improve their models — but we never know. Prefer to keep things private? Just fill the form manually — no AI involved.', + 'Heads up: we use Google AI to extract your info. As we pay for the service, Google should not use this content to improve their models — but we never know. Prefer to keep things fully internal? Just fill the form manually — no AI involved.', )}
)} + ))} + + {/* Both panels stay mounted and the inactive one is hidden, rather than swapped out. + Unmounting the recorder would tear down an in-progress recording and drop any transcript + corrections the moment someone peeked at the other tab. `display: none` also keeps the + hidden panel out of the flex layout and away from assistive tech. */} +
+ { + await runExtraction({content: transcript, source: 'voice'}) + }} + isExtracting={isExtracting} + isSubmitting={isSubmitting} + progress={extractionProgress} + /> +
+
+ +
{extractionError && (

{extractionError}

)} + {showPhotoReminder && ( +
+ + +

+ {t( + 'profile.optional.photo_reminder', + 'One thing we cannot fill in for you: your photos. They are still an important thing people look at (at least just to visually identify you), so do not leave without adding one or two.', + )} +

+ + +
+ )} {/* The rule that used to sit here (`border border-b`, which draws two) is now owned by `Category` itself, so every section boundary is drawn the same way. */} - + {/*
*/} diff --git a/web/components/voice-autofill-section.tsx b/web/components/voice-autofill-section.tsx new file mode 100644 index 00000000..e3ec200b --- /dev/null +++ b/web/components/voice-autofill-section.tsx @@ -0,0 +1,348 @@ +import { + ArrowPathIcon, + MicrophoneIcon, + PauseIcon, + PlayIcon, + StopIcon, +} from '@heroicons/react/24/solid' +import clsx from 'clsx' +import {useEffect, useState} from 'react' +import Textarea from 'react-expanding-textarea' +import {Button} from 'web/components/buttons/button' +import {Col} from 'web/components/layout/col' +import {Row} from 'web/components/layout/row' +import {AudioPlayer} from 'web/components/widgets/audio-player' +import {useAudioRecorder} from 'web/hooks/use-audio-recorder' +import {useT} from 'web/lib/locale' +import { + clearPendingRecording, + loadPendingRecording, + savePendingRecording, + savePendingTranscript, +} from 'web/lib/util/recording-store' + +function formatTime(totalSeconds: number) { + const minutes = Math.floor(totalSeconds / 60) + const seconds = totalSeconds % 60 + return `${minutes}:${String(seconds).padStart(2, '0')}` +} + +/** + * Talking prompts. Deliberately open-ended and unordered — the point is to unblock someone staring + * at a mic button, not to turn speaking into another form to fill in. + */ +function useTalkingPoints() { + const t = useT() + return [ + t('profile.voice.prompt.who', "What's your gender, how old are you, and where do you live?"), + t('profile.voice.prompt.work', 'What do you do — work, studies, projects?'), + t('profile.voice.prompt.time', 'How do you like to spend your time?'), + t('profile.voice.prompt.care', 'What do you care about most?'), + t('profile.voice.prompt.looking', 'What kind of connection are you looking for?'), + ] +} + +export function VoiceAutofillSection(props: { + /** Runs transcription; resolves to the transcript, or null when it failed. */ + onTranscribe: (blob: Blob) => Promise + /** Hands the (possibly edited) transcript to the profile extractor. */ + onExtract: (transcript: string) => Promise + isExtracting: boolean + isSubmitting: boolean + progress: number +}) { + const {onTranscribe, onExtract, isExtracting, isSubmitting, progress} = props + const t = useT() + const talkingPoints = useTalkingPoints() + const recorder = useAudioRecorder({ + onRecorded: (recorded, recordedSeconds) => { + void savePendingRecording({ + blob: recorded, + mimeType: recorded.type, + seconds: recordedSeconds, + transcript: null, + }) + }, + }) + const {isRecording, isPaused, seconds, level, blob, previewUrl, restore} = recorder + + const [isTranscribing, setIsTranscribing] = useState(false) + // Kept in state (not just handed straight to the extractor) so the user can read what we heard and + // fix any mis-transcribed names or places before it becomes their profile. + const [transcript, setTranscript] = useState(null) + + // Rehydrate whatever was left behind by a reload, an app close, or an earlier session. Runs once; + // `restore` no-ops if a recording is already under way. + useEffect(() => { + let cancelled = false + loadPendingRecording().then((pending) => { + if (cancelled || !pending) return + restore(pending.blob, pending.seconds) + if (pending.transcript) setTranscript(pending.transcript) + }) + return () => { + cancelled = true + } + }, [restore]) + + const busy = isTranscribing || isExtracting || isSubmitting + + const handleTranscribe = async () => { + if (!blob) return + setIsTranscribing(true) + try { + const text = await onTranscribe(blob) + if (text) { + setTranscript(text) + // Saved alongside the audio so a reload does not force a second (paid) transcription. + void savePendingTranscript(text) + } + } finally { + setIsTranscribing(false) + } + } + + const handleExtract = async () => { + if (!transcript?.trim()) return + await onExtract(transcript.trim()) + // The recording has served its purpose; the extracted fields are the artefact now. + setTranscript(null) + recorder.reset() + void clearPendingRecording() + } + + const startOver = () => { + setTranscript(null) + recorder.reset() + void clearPendingRecording() + } + + const errorMessage = + recorder.error === 'permission' + ? t( + 'profile.voice.error.permission', + 'We could not access your microphone. Allow microphone access in your browser, then try again.', + ) + : recorder.error === 'unsupported' + ? t( + 'profile.voice.error.unsupported', + 'Your browser does not support voice recording. Try the link or text option instead.', + ) + : recorder.error + ? t('profile.voice.error.failed', 'Recording failed. Please try again.') + : null + + return ( + +
+ {t( + 'profile.voice.description', + 'Rather talk than type? Record yourself for a few minutes and we will fill in your profile from what you say.', + )} +
+
+ {t( + 'profile.voice.guidance', + 'Heads up: your recording is sent to OpenAI Whisper to be transcribed, and the transcript to Google AI to fill in the fields. We pay for both services, so neither should use your content to train their models — but we never know. Prefer to keep things fully internal? Just fill the form manually — no AI involved.', + )} +
+ + {transcript === null ? ( + <> + {/* Prompts and mic sit side by side from `sm` up: stacked, they pushed the mic below the + fold on short viewports. Talking points stay visible while recording — they are the + script. On mobile they stack, mic underneath. */} + +
+

+ {t('profile.voice.prompts.title', 'Talk freely, or use these as a starting point:')} +

+
    + {talkingPoints.map((point) => ( +
  • + + • + + {point} +
    • + ))} +
+

+ {t( + 'profile.voice.prompts.footer', + 'No need to be polished — we clean up the ums and false starts. Skip anything you would rather not share.', + )} +

+
+ + {!blob && ( + +
+ {isRecording && !isPaused && ( + <> + + + + )} + +
+ + {isRecording ? ( + <> + + {formatTime(seconds)} + + + {isPaused + ? t('profile.voice.paused', 'Paused') + : t('profile.voice.recording', 'Recording…')} + + {/* Wraps rather than overflowing the narrow right-hand column. */} + + + + + + ) : ( + + {t('profile.voice.tap_to_record', 'Tap to start recording')} + + )} + + )} + + + {blob && !isRecording && ( + +

+ {t( + 'profile.voice.review_recording', + 'Feel free to listen before we transcribe it — {duration} recorded.', + ).replace('{duration}', formatTime(seconds))} +

+ {previewUrl && } + + + + + + )} + + ) : ( + +

+ {t( + 'profile.voice.review_transcript', + 'Here is what we heard. No need to tidy it up — the ums, the false starts, the times you changed your mind mid-sentence are all handled for you, and none of it ends up on your profile. Only worth fixing is what we misheard: names and places are the usual suspects.', + )} +

+