Add voice recording, transcription, and audio player components

- Implemented `AudioPlayer` for custom audio playback controls.
- Added `useAudioRecorder` hook for managing microphone audio recording with visual feedback.
- Introduced audio transcription API integration using OpenAI Whisper.
- Created `blobToBase64` utility to encode recorded blobs for API consumption.
- Added IndexedDB-based `recording-store` for saving in-progress recordings.
- Developed `VoiceAutofillSection` for guided recording, playback, transcription, and editing.
This commit is contained in:
MartinBraquet
2026-07-25 18:01:58 +02:00
parent e8134fcc14
commit 4c76b643e3
19 changed files with 1588 additions and 41 deletions

View File

@@ -11,8 +11,8 @@ android {
applicationId "com.compassconnections.app"
minSdkVersion rootProject.ext.minSdkVersion
targetSdkVersion rootProject.ext.targetSdkVersion
versionCode 131
versionName "1.31.0"
versionCode 132
versionName "1.32.0"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
aaptOptions {
// Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps.

View File

@@ -92,6 +92,7 @@ import {setLastOnlineTime} from './set-last-online-time'
import {shipProfiles} from './ship-profiles'
import {starProfile} from './star-profile'
import {stats} from './stats'
import {transcribeAudio} from './transcribe-audio'
import {unsubscribe} from './unsubscribe'
import {updateEvent} from './update-event'
import {updateMe} from './update-me'
@@ -653,6 +654,7 @@ const handlers: {[k in APIPath]: APIHandler<k>} = {
vote: vote,
'validate-username': validateUsernameEndpoint,
'llm-extract-profile': llmExtractProfileEndpoint,
'transcribe-audio': transcribeAudio,
// 'user/:username': getUser,
// 'user/:username/lite': getDisplayUser,
// 'user/by-id/:id/lite': getDisplayUser,
@@ -678,7 +680,8 @@ Object.entries(handlers).forEach(([path, handler]) => {
const apiRoute = [
url,
express.json({limit: '1mb'}),
// Endpoints that carry binary payloads (e.g. base64 audio) declare a larger `bodyLimit`.
express.json({limit: (api as any).bodyLimit ?? '1mb'}),
allowCorsUnrestricted,
cache,
typedEndpoint(path as any, handler as any),

View File

@@ -9,6 +9,8 @@ import {
GENDERS,
LANGUAGE_CHOICES,
MBTI_CHOICES,
NEUROTYPE_CHOICES,
ORIENTATION_CHOICES,
POLITICAL_CHOICES,
PSYCHEDELICS_CHOICES,
RACE_CHOICES,
@@ -23,7 +25,7 @@ import {debug} from 'common/logger'
import {ProfileWithoutUser} from 'common/profiles/profile'
import {SITE_ORDER} from 'common/socials'
import {removeNullOrUndefinedProps} from 'common/util/object'
import {parseJsonContentToText} from 'common/util/parse'
import {parseJsonContentToText, textToJSONContent} from 'common/util/parse'
import {HOUR_MS, MINUTE_MS, sleep} from 'common/util/time'
import {createHash} from 'crypto'
import {promises as fs} from 'fs'
@@ -38,18 +40,26 @@ const CACHE_DIR = join(tmpdir(), 'compass-llm-cache')
const CACHE_TTL_MS = 24 * HOUR_MS
const PROCESSING_TTL_MS = 10 * MINUTE_MS
type ExtractSource = 'text' | 'url' | 'voice'
interface ParsedBody {
content?: string
url?: string
locale?: string
source?: ExtractSource
}
// Bump whenever the extraction prompt changes. The cache key is otherwise derived purely from the
// request, so a prompt fix would keep returning the old answer for the 24h TTL — which looks exactly
// like the fix not working.
const PROMPT_VERSION = 2
function getCacheKey(parsedBody: ParsedBody): string {
if (!USE_CACHE) return ''
const hash = createHash('sha256')
// Normalize: sort keys for consistent hashing
const normalized = JSON.stringify(parsedBody, Object.keys(parsedBody).sort())
hash.update(normalized)
hash.update(`v${PROMPT_VERSION}:${normalized}`)
return hash.digest('hex')
}
@@ -79,6 +89,8 @@ async function validateProfileFields(
'cannabis_intention',
'psychedelics_pref',
'cannabis_pref',
'orientation',
'neurotype',
]
for (const key of toArray) {
if (result[key] !== undefined) {
@@ -113,6 +125,10 @@ async function validateProfileFields(
'occupation_title',
'religious_beliefs',
'political_details',
'gender_details',
'orientation_details',
'neurotype_details',
'accessibility_notes',
]
for (const key of toString) {
if (result[key] !== undefined) {
@@ -246,7 +262,10 @@ async function setCachedResult(cacheKey: string, result: any): Promise<void> {
await fs.mkdir(CACHE_DIR, {recursive: true})
const cacheFile = join(CACHE_DIR, `${cacheKey}.json`)
await fs.writeFile(cacheFile, JSON.stringify(result), 'utf-8')
debug('Cached LLM result', {cacheKey: cacheKey.substring(0, 8), result})
debug('Cached LLM result', {
cacheKey: cacheKey.substring(0, 8),
result: JSON.stringify(result),
})
} catch (error) {
log('Failed to write cache', {cacheKey, error})
// Don't throw - caching failure shouldn't break the main flow
@@ -296,11 +315,13 @@ async function processAndCache(
content?: string | undefined,
url?: string | undefined,
locale?: string,
source?: ExtractSource,
): Promise<void> {
log('Extracting profile from content', {
contentLength: content?.length,
url,
locale,
source,
})
try {
let bio: JSONContent | undefined
@@ -309,7 +330,7 @@ async function processAndCache(
debug(JSON.stringify(bio, null, 2))
content = parseJsonContentToText(bio)
}
const profile = await callLLM(content, locale)
const profile = await callLLM(content, locale, source)
if (bio) {
profile.bio = bio
}
@@ -422,7 +443,9 @@ async function _callClaude(text: string) {
export async function callLLM(
content: string,
locale?: string,
source?: ExtractSource,
): Promise<Partial<ProfileWithoutUser>> {
const isVoice = source === 'voice'
const [INTERESTS, CAUSE_AREAS, WORK_AREAS] = await Promise.all([
getOptions('interests', locale),
getOptions('causes', locale),
@@ -451,12 +474,19 @@ export async function callLLM(
cannabis_intention: Object.values(SUBSTANCE_INTENTION_CHOICES),
psychedelics_pref: Object.values(SUBSTANCE_PREFERENCE_CHOICES),
cannabis_pref: Object.values(SUBSTANCE_PREFERENCE_CHOICES),
orientation: Object.values(ORIENTATION_CHOICES),
neurotype: Object.values(NEUROTYPE_CHOICES),
}
const PROFILE_FIELDS: Partial<Record<keyof ProfileWithoutUser, any>> = {
// Basic info
age: 'Number. Age in years (between 18 and 100).',
gender: `String. One of: ${validChoices.pref_gender?.join(', ')}. If multiple mentioned, use the most likely one. Infer if you have enough evidence`,
gender_details:
'String. Free-form elaboration on their gender identity, only if they say more than the label itself.',
orientation: `Array. Any of: ${validChoices.orientation?.join(', ')}. Only if stated — never infer from the gender of a partner or of who they are looking for.`,
orientation_details:
'String. Free-form elaboration on their orientation, only if they say more than the label itself.',
height_in_inches: 'Number. Height converted to inches.',
city: 'String. Current city of residence (English spelling).',
country: 'String. Current country of residence (English spelling).',
@@ -498,6 +528,11 @@ export async function callLLM(
big5_agreeableness: 'Number 0100. Only if explicitly self-reported, never infer.',
big5_neuroticism: 'Number 0100. Only if explicitly self-reported, never infer.',
// Neurotype is an identity here, not a diagnosis: only ever take the person's own words for it.
neurotype: `Array. Any of: ${validChoices.neurotype?.join(', ')}. Only if they identify this way themselves — never infer it from how they describe their personality, focus, energy or social life.`,
neurotype_details:
'String. Free-form elaboration on their neurotype, only if they say more than the label itself.',
// Beliefs
religion: `Array. Any of: ${validChoices.religion?.join(', ')}`,
religious_beliefs:
@@ -511,7 +546,7 @@ export async function callLLM(
'Number. Minimum preferred age of match (higher than 18, only if mentioned, do NOT infer).',
pref_age_max:
'Number. Maximum preferred age of match (lower than 100, only if mentioned, do NOT infer).',
pref_gender: `Array. Any of: ${validChoices.pref_gender?.join(', ')}`,
pref_gender: `Array. Any of: ${validChoices.pref_gender?.join(', ')}. Only the genders they actually name as sought. If they say gender does not matter to them, or is unimportant to their attraction, OMIT this field — an omitted field already means "no preference", so listing every option instead is both wrong and unreadable.`,
pref_relation_styles: `Array. Any of: ${validChoices.pref_relation_styles?.join(', ')}`,
pref_romantic_styles: `Array. Any of: ${validChoices.pref_romantic_styles?.join(', ')}`,
relationship_status: `Array. Any of: ${validChoices.relationship_status?.join(', ')}`,
@@ -523,6 +558,8 @@ export async function callLLM(
headline:
'String. Summary of who they are, in their own voice (first person). Maximum 200 characters total. Cannot be null.',
keywords: 'Array of 36 short tags summarising the person.',
accessibility_notes:
'String. Practical things that help someone meet them well — access needs, energy levels, sensory preferences, venue preferences. Only if mentioned; never infer a disability, and keep their own framing and wording.',
links: `Object. Key is any of: ${SITE_ORDER.join(', ')}.`,
// Taxonomies — match existing labels first, only add new if truly no close match exists
@@ -531,7 +568,25 @@ export async function callLLM(
work: `Array. Use only existing labels, do not add new if no close match. Any of: ${validChoices.work?.join(', ')}`,
}
const EXTRACTION_PROMPT = `You are a profile information extraction expert analyzing text from a personal webpage, bio, or similar source.
// For text and URL sources the bio is the source material itself, stored verbatim. A speech
// transcript makes a poor bio (filler words, false starts, no paragraphs), so for voice we ask
// the model to write it instead.
if (isVoice) {
PROFILE_FIELDS.bio =
'String. A first-person bio written from what the person said, in their own voice and their ' +
'own language. Keep their wording and personality wherever you can; only clean up filler ' +
'words, false starts, repetitions and transcription noise, and organise it into ' +
'paragraphs. Separate each paragraph from the next with a BLANK LINE, i.e. two newline ' +
'characters ("\\n\\n") — a single newline is not enough. Never add facts, opinions or ' +
'flourishes they did not say, and never write about them in the third person. Plain text ' +
'only — no markdown.'
}
const EXTRACTION_PROMPT = `You are a profile information extraction expert analyzing ${
isVoice
? 'a speech-to-text transcript of someone talking about themselves out loud'
: 'text from a personal webpage, bio, or similar source'
}.
TASK: Extract structured profile data and return it as a single valid JSON object.
@@ -540,12 +595,20 @@ RULES:
- Omit the key in the output for missing fields
- For taxonomy fields (interests, causes, work): match existing labels first; only add a new label if truly no existing one is close
- For big5 scores: only populate if the person explicitly states a test result — never infer from personality description
- Return valid JSON only — no markdown, no explanation, no extra text
- Never answer a multi-choice field by selecting every option it offers. An expression of openness or indifference ("gender doesn't matter to me", "I'm open to anything") is not a selection of all values — omit the field, which already means "no preference"
- Return valid JSON only — no markdown, no explanation, no extra text${
isVoice
? `
- The transcript is spoken language: expect filler words, false starts, self-corrections and speech-recognition errors. Read past them, and when the person corrects themselves keep the corrected version
- Ignore anything the person says to the recorder rather than about themselves (e.g. "let me start over", "what else should I say")
- Spoken numbers, places and names may be mis-transcribed; only fill a field when you are confident what was meant`
: ''
}
SCHEMA (each value describes the expected type and accepted values):
${JSON.stringify(PROFILE_FIELDS, null, 2)}
TEXT TO ANALYZE:
${isVoice ? 'TRANSCRIPT TO ANALYZE' : 'TEXT TO ANALYZE'}:
`
const text = EXTRACTION_PROMPT + content
if (text.length > MAX_CONTEXT_LENGTH) {
@@ -564,6 +627,10 @@ TEXT TO ANALYZE:
try {
parsed = typeof outputText === 'string' ? JSON.parse(outputText) : outputText
parsed = await validateProfileFields(parsed, validChoices)
// The bio column holds rich text; the model answers with plain prose.
if (typeof parsed.bio === 'string') {
parsed.bio = parsed.bio.trim() ? textToJSONContent(parsed.bio) : undefined
}
parsed = removeNullOrUndefinedProps(parsed)
} catch (parseError) {
log('Failed to parse LLM response as JSON', {outputText, parseError})
@@ -647,7 +714,7 @@ export async function fetchOnlineProfile(url: string | undefined): Promise<JSONC
}
export const llmExtractProfileEndpoint: APIHandler<'llm-extract-profile'> = async (parsedBody) => {
const {url, locale} = parsedBody
const {url, locale, source} = parsedBody
const content = parsedBody.content
if (content && url) {
@@ -672,7 +739,7 @@ export const llmExtractProfileEndpoint: APIHandler<'llm-extract-profile'> = asyn
await setProcessing(cacheKey)
// Kick off async processing (don't await)
processAndCache(cacheKey, content, url, locale).catch((err) => {
processAndCache(cacheKey, content, url, locale, source).catch((err) => {
log('Unexpected error in async processing', {cacheKey, error: err})
})

View File

@@ -0,0 +1,100 @@
import {APIErrors, APIHandler} from 'api/helpers/endpoint'
import {debug} from 'common/logger'
import {log} from 'shared/monitoring/log'
// Whisper's own hard limit is 25 MB. We cap below that so we reject with a friendly message rather
// than letting OpenAI 413 on us. Opus runs ~200-400 KB/minute, so this is many minutes of speech.
const MAX_AUDIO_BYTES = 20 * 1024 * 1024
// `gpt-4o-transcribe` is Whisper's successor and the default in our other voice codebase; set
// OPENAI_TRANSCRIBE_MODEL=whisper-1 to fall back to classic Whisper.
const TRANSCRIBE_MODEL = process.env.OPENAI_TRANSCRIBE_MODEL || 'gpt-4o-transcribe'
// Container → file extension. OpenAI infers the codec from the uploaded filename, so this has to be
// right; `;codecs=...` suffixes that browsers append are stripped first.
const EXTENSION_BY_MIME: Record<string, string> = {
'audio/webm': 'webm',
'audio/ogg': 'ogg',
'audio/mp4': 'mp4',
'audio/m4a': 'm4a',
'audio/x-m4a': 'm4a',
'audio/mpeg': 'mp3',
'audio/mp3': 'mp3',
'audio/wav': 'wav',
'audio/x-wav': 'wav',
'audio/flac': 'flac',
}
function parseMimeType(mimeType: string): {contentType: string; extension: string} {
const base = mimeType.split(';')[0].trim().toLowerCase()
const extension = EXTENSION_BY_MIME[base]
if (extension) return {contentType: base, extension}
// Unknown container: webm/opus is what every Chromium browser records, so it is the safest guess.
log('Unsupported audio mime type, falling back to audio/webm', {mimeType})
return {contentType: 'audio/webm', extension: 'webm'}
}
// Whisper takes an ISO-639-1 code; our locales are either 'en' or 'en-US'-shaped.
function toLanguageCode(locale: string | undefined): string | undefined {
const code = locale?.split(/[-_]/)[0]?.toLowerCase()
return code && /^[a-z]{2}$/.test(code) ? code : undefined
}
export const transcribeAudio: APIHandler<'transcribe-audio'> = async (props) => {
const {audio, mimeType, locale} = props
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) {
log('OPENAI_API_KEY not configured')
throw APIErrors.internalServerError('Voice transcription is not configured')
}
// `audio` is base64: 4 characters encode 3 bytes.
const sizeInBytes = Math.floor((audio.length * 3) / 4)
if (sizeInBytes > MAX_AUDIO_BYTES) {
throw APIErrors.badRequest('Recording is too long. Please record a shorter message.')
}
const {contentType, extension} = parseMimeType(mimeType)
const language = toLanguageCode(locale)
log('Transcribing audio', {mimeType, contentType, sizeInBytes, model: TRANSCRIBE_MODEL, language})
const form = new FormData()
form.append(
'file',
new Blob([Buffer.from(audio, 'base64')], {type: contentType}),
`voice.${extension}`,
)
form.append('model', TRANSCRIBE_MODEL)
// Punctuated, readable text — the extraction step downstream reads much better prose than a
// single unbroken run of words.
form.append('response_format', 'text')
if (language) form.append('language', language)
let response: Response
try {
response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
method: 'POST',
headers: {Authorization: `Bearer ${apiKey}`},
body: form,
})
} catch (error) {
log('OpenAI transcription request failed', {error})
throw APIErrors.internalServerError('Failed to transcribe the recording')
}
if (!response.ok) {
const errorText = await response.text()
log('OpenAI transcription API error', {status: response.status, error: errorText})
throw APIErrors.internalServerError('Failed to transcribe the recording')
}
const transcript = (await response.text()).trim()
debug({transcript})
if (!transcript) {
throw APIErrors.badRequest('We could not hear any speech in that recording. Please try again.')
}
return {transcript}
}

View File

@@ -1050,6 +1050,35 @@
"profile.llm.extract.description": "Füllen Sie Ihr Profil automatisch aus, indem Sie einen Link (Notion, Google Docs, persönliche Website usw.) einfügen oder Ihren Inhalt direkt einfügen.",
"profile.llm.extract.guidance": "Achtung: Wir verwenden Google AI, um Ihre Informationen zu extrahieren. Google kann diesen Inhalt verwenden, um seine Modelle zu verbessern. Bevorzugen Sie Privatsphäre? Füllen Sie das Formular einfach manuell aus — keine KI beteiligt.",
"profile.llm.extract.placeholder": "Fügen Sie eine URL ein oder fügen Sie hier Ihren Profilinhalt ein.",
"profile.autofill.by_voice": "Per Sprache",
"profile.autofill.by_link": "Per Link oder Text",
"profile.voice.description": "Sprechen Sie lieber, als zu schreiben? Nehmen Sie sich ein bis zwei Minuten auf, und wir füllen Ihr Profil aus dem aus, was Sie sagen.",
"profile.voice.guidance": "Achtung: Ihre Aufnahme wird zur Transkription an OpenAI Whisper geschickt und das Transkript an Google AI, um die Felder auszufüllen. Wir zahlen für beide Dienste, daher sollte keiner Ihre Inhalte zum Trainieren seiner Modelle verwenden — aber wissen kann man es nie. Bevorzugen Sie Privatsphäre? Füllen Sie das Formular einfach manuell aus — keine KI beteiligt.",
"profile.voice.prompts.title": "Sprechen Sie frei, oder nehmen Sie diese Fragen als Ausgangspunkt:",
"profile.voice.prompt.who": "Wer sind Sie, und wo leben Sie?",
"profile.voice.prompt.work": "Was machen Sie — Arbeit, Studium, Projekte?",
"profile.voice.prompt.time": "Wie verbringen Sie Ihre Zeit am liebsten?",
"profile.voice.prompt.care": "Was ist Ihnen am wichtigsten?",
"profile.voice.prompt.looking": "Welche Art von Verbindung suchen Sie?",
"profile.voice.prompts.footer": "Es muss nicht perfekt sein — wir räumen Ähs und Fehlstarts auf. Lassen Sie weg, was Sie nicht teilen möchten.",
"profile.voice.tap_to_record": "Zum Aufnehmen tippen",
"profile.voice.start": "Aufnahme starten",
"profile.voice.stop": "Aufnahme beenden",
"profile.voice.recording": "Aufnahme läuft…",
"profile.voice.paused": "Pausiert",
"profile.voice.pause": "Pausieren",
"profile.voice.resume": "Fortsetzen",
"profile.voice.done_talking": "Ich bin fertig",
"profile.voice.review_recording": "Hören Sie es sich vor der Transkription an — {duration} aufgenommen.",
"profile.voice.transcribe": "Meine Aufnahme transkribieren",
"profile.voice.transcribing": "Transkribiere…",
"profile.voice.record_again": "Neu aufnehmen",
"profile.voice.review_transcript": "Das haben wir gehört. Sie müssen nichts aufräumen: Ähs, Fehlstarts und die Stellen, an denen Sie sich mitten im Satz korrigiert haben, übernehmen wir für Sie — nichts davon landet in Ihrem Profil. Zu korrigieren lohnt sich nur, was wir falsch verstanden haben: Namen und Orte sind die üblichen Verdächtigen.",
"profile.voice.fill_profile": "Mein Profil ausfüllen",
"profile.voice.error.permission": "Wir konnten nicht auf Ihr Mikrofon zugreifen. Erlauben Sie den Mikrofonzugriff in Ihrem Browser, und versuchen Sie es erneut.",
"profile.voice.error.unsupported": "Ihr Browser unterstützt keine Sprachaufnahme. Nutzen Sie stattdessen die Option Link oder Text.",
"profile.voice.error.failed": "Die Aufnahme ist fehlgeschlagen. Bitte versuchen Sie es erneut.",
"profile.voice.error.transcription": "Wir konnten Ihre Aufnahme nicht transkribieren. Sie ist noch da — versuchen Sie es erneut, oder füllen Sie Ihr Profil manuell aus.",
"profile.mbti": "MBTI",
"profile.mbti.INTJ": "Architekt",
"profile.mbti.INTP": "Logiker",
@@ -1130,6 +1159,8 @@
"profile.optional.og_card": "Profilkarte",
"profile.optional.orientation": "Sexuelle Orientierung",
"profile.optional.photos": "Fotos",
"profile.optional.photo_reminder": "Eines können wir nicht für Sie ausfüllen: Ihre Fotos. Sie sind das Erste, was die Leute ansehen — gehen Sie also nicht, ohne ein paar hinzuzufügen.",
"profile.optional.photo_reminder.cta": "Zu den Fotos",
"profile.optional.political_beliefs": "Politische Ansichten",
"profile.optional.raised_in": "Ort, an dem ich aufgewachsen bin",
"profile.optional.raised_in_hint": "Besonders nützlich, wenn Sie in einem anderen Land aufgewachsen sind als dem, in dem Sie jetzt leben und wenn dies Ihre kulturellen Referenzen, Werte und Lebenserfahrungen widerspiegelt.",

View File

@@ -1049,6 +1049,35 @@
"profile.llm.extract.description": "Remplissez automatiquement votre profil en déposant un lien (Notion, Google Docs, site personnel, etc.) ou en collant directement votre contenu.",
"profile.llm.extract.guidance": "Attention : nous utilisons Google AI pour extraire vos informations. Comme nous payons pour leur service, Google ne devrait pas utiliser ce contenu pour améliorer ses modèles — mais on ne sait jamais. Vous préférez garder les choses privées ? Remplissez simplement le formulaire manuellement — aucune IA impliquée.",
"profile.llm.extract.placeholder": "Insérez une URL ou collez le contenu de votre profil ici.",
"profile.autofill.by_voice": "À la voix",
"profile.autofill.by_link": "Par lien ou texte",
"profile.voice.description": "Vous préférez parler plutôt qu'écrire ? Enregistrez-vous pendant une ou deux minutes et nous remplirons votre profil à partir de ce que vous dites.",
"profile.voice.guidance": "Attention : votre enregistrement est envoyé à OpenAI Whisper pour être transcrit, et la transcription à Google AI pour remplir les champs. Comme nous payons pour ces services, ni l'un ni l'autre ne devrait utiliser votre contenu pour entraîner ses modèles — mais on ne sait jamais. Vous préférez garder les choses privées ? Remplissez simplement le formulaire manuellement — aucune IA impliquée.",
"profile.voice.prompts.title": "Parlez librement, ou partez de ces questions :",
"profile.voice.prompt.who": "Qui êtes-vous, et où vivez-vous ?",
"profile.voice.prompt.work": "Que faites-vous — travail, études, projets ?",
"profile.voice.prompt.time": "Comment aimez-vous passer votre temps ?",
"profile.voice.prompt.care": "Qu'est-ce qui vous tient le plus à cœur ?",
"profile.voice.prompt.looking": "Quel type de connexion recherchez-vous ?",
"profile.voice.prompts.footer": "Pas besoin d'être parfait — nous nettoyons les hésitations et les faux départs. Passez ce que vous préférez ne pas partager.",
"profile.voice.tap_to_record": "Appuyez pour enregistrer",
"profile.voice.start": "Démarrer l'enregistrement",
"profile.voice.stop": "Arrêter l'enregistrement",
"profile.voice.recording": "Enregistrement…",
"profile.voice.paused": "En pause",
"profile.voice.pause": "Mettre en pause",
"profile.voice.resume": "Reprendre",
"profile.voice.done_talking": "J'ai terminé",
"profile.voice.review_recording": "Écoutez-vous avant la transcription — {duration} enregistré.",
"profile.voice.transcribe": "Transcrire mon enregistrement",
"profile.voice.transcribing": "Transcription…",
"profile.voice.record_again": "Réenregistrer",
"profile.voice.review_transcript": "Voici ce que nous avons entendu. Pas besoin de le nettoyer : les hésitations, les faux départs, les fois où vous avez changé d'avis en cours de phrase sont gérés pour vous, et rien de tout cela ne se retrouvera sur votre profil. La seule chose à corriger, c'est ce que nous avons mal entendu : les noms et les lieux sont les suspects habituels.",
"profile.voice.fill_profile": "Remplir mon profil",
"profile.voice.error.permission": "Nous n'avons pas pu accéder à votre microphone. Autorisez l'accès au microphone dans votre navigateur, puis réessayez.",
"profile.voice.error.unsupported": "Votre navigateur ne prend pas en charge l'enregistrement vocal. Essayez plutôt l'option lien ou texte.",
"profile.voice.error.failed": "L'enregistrement a échoué. Veuillez réessayer.",
"profile.voice.error.transcription": "Nous n'avons pas pu transcrire votre enregistrement. Il est toujours là — réessayez, ou remplissez votre profil manuellement.",
"profile.mbti": "MBTI",
"profile.mbti.INTJ": "Architecte",
"profile.mbti.INTP": "Logicien",
@@ -1129,6 +1158,8 @@
"profile.optional.og_card": "Carte de profil",
"profile.optional.orientation": "Orientation sexuelle",
"profile.optional.photos": "Photos",
"profile.optional.photo_reminder": "Une chose que nous ne pouvons pas remplir à votre place : vos photos. C'est la première chose que les gens regardent, alors ne partez pas sans en ajouter quelques-unes.",
"profile.optional.photo_reminder.cta": "Aller aux photos",
"profile.optional.political_beliefs": "Opinions politiques",
"profile.optional.raised_in": "Lieu où j'ai grandi",
"profile.optional.raised_in_hint": "Particulièrement utile si vous avez grandi dans un pays différent de celui où vous vivez maintenant et si cela reflète vos références culturelles, vos valeurs et vos expériences de vie.",

View File

@@ -55,6 +55,13 @@ export type APIGenericSchema = {
*/
rateLimited?: boolean
/**
* Maximum request body size, as an `express.json({limit})` value (e.g. '20mb').
* Only needed for endpoints carrying binary payloads such as base64-encoded audio.
* @default '1mb'
*/
bodyLimit?: string
/**
* Zod schema for request validation
* - For GET requests: Validates query parameters
@@ -1285,6 +1292,10 @@ export const API = (_apiTypeCheck = {
content: z.string().min(1).optional(),
url: z.string().url().optional(),
locale: z.string().optional(),
// Where `content` came from. 'voice' means it is a speech transcript, so the LLM has to be
// told to read past filler words and to write the bio itself rather than us storing the
// transcript verbatim.
source: z.enum(['text', 'url', 'voice']).optional(),
})
.strict(),
returns: {} as {
@@ -1294,6 +1305,27 @@ export const API = (_apiTypeCheck = {
summary: 'Extract profile information from text using LLM',
tag: 'Profiles',
},
'transcribe-audio': {
method: 'POST',
authed: true,
rateLimited: true,
// Audio arrives base64-encoded in the JSON body, which is far bigger than the default 1mb.
bodyLimit: '20mb',
props: z
.object({
// Base64-encoded audio (no data: URI prefix).
audio: z.string().min(1),
// Recording container/codec as reported by MediaRecorder, e.g. 'audio/webm;codecs=opus'.
mimeType: z.string().min(1),
locale: z.string().optional(),
})
.strict(),
returns: {} as {
transcript: string
},
summary: 'Transcribe a voice recording to text',
tag: 'Profiles',
},
'get-user-journeys': {
method: 'GET',
authed: true,

View File

@@ -30,6 +30,7 @@ export const secrets = (
'DB_ENC_MASTER_KEY_BASE64',
'GOOGLE_CLIENT_SECRET',
'GEMINI_API_KEY',
'OPENAI_API_KEY',
// Some typescript voodoo to keep the string literal types while being not readonly.
] as const
).concat()

View File

@@ -83,6 +83,61 @@ export function parseJsonContentToText(content: JSONContent | string | undefined
return typeof content === 'string' ? content : richTextToString(content)
}
/**
* Inverse of `parseJsonContentToText`: turns plain text into a minimal Tiptap document. Used when
* an LLM hands us prose (e.g. a bio written from a voice transcript) that has to be stored in a
* rich-text column.
*
* A blank line always starts a new paragraph, and single newlines inside such a block are hard
* breaks. But when the text contains no blank line at all, single newlines are the only structure
* on offer, so they are read as paragraph breaks instead — LLMs routinely separate paragraphs with
* one `\n` however firmly the prompt asks for two, and the alternative is one giant paragraph.
*/
export function textToJSONContent(text: string): JSONContent {
const separator = /\n\s*\n/.test(text) ? /\n\s*\n+/ : '\n'
const paragraphs = text
.split(separator)
.map((block) => block.trim())
.filter((block) => block.length > 0)
return {
type: 'doc',
content: paragraphs.length
? paragraphs.map((block) => ({
type: 'paragraph',
// Single newlines inside a block are line breaks, not paragraph breaks.
content: block
.split('\n')
.flatMap((line, i) => [
...(i > 0 ? [{type: 'hardBreak'}] : []),
...(line.trim() ? [{type: 'text', text: line.trim()}] : []),
]),
}))
: [{type: 'paragraph'}],
}
}
/**
* Appends one rich-text document to another, so a second pass adds to what is already written
* instead of replacing it (e.g. recording a follow-up voice note to extend an existing bio).
*
* Each side is trimmed of its leading and trailing empty paragraphs first, so the join does not
* accumulate blank space every time something is appended.
*/
export function concatJSONContent(
first: JSONContent | null | undefined,
second: JSONContent | null | undefined,
): JSONContent {
const blocksOf = (doc: JSONContent | null | undefined) => {
if (!doc) return []
const cleaned = cleanDoc(doc)
return Array.isArray(cleaned.content) ? cleaned.content : []
}
const content = [...blocksOf(first), ...blocksOf(second)]
return {type: 'doc', content: content.length ? content : [{type: 'paragraph'}]}
}
export function urlBase64ToUint8Array(base64String: string) {
const padding = '='.repeat((4 - (base64String.length % 4)) % 4)
const base64 = (base64String + padding).replace(/-/g, '+').replace(/_/g, '/')

View File

@@ -0,0 +1,85 @@
import {concatJSONContent, parseJsonContentToText, textToJSONContent} from 'common/util/parse'
describe('textToJSONContent', () => {
it('makes one paragraph per blank-line-separated block', () => {
expect(textToJSONContent('First para.\n\nSecond para.')).toEqual({
type: 'doc',
content: [
{type: 'paragraph', content: [{type: 'text', text: 'First para.'}]},
{type: 'paragraph', content: [{type: 'text', text: 'Second para.'}]},
],
})
})
// The LLM writing our bios routinely uses one '\n' between paragraphs no matter how firmly the
// prompt asks for two; without this the whole bio collapsed into a single paragraph.
it('treats single newlines as paragraph breaks when there is no blank line anywhere', () => {
expect(textToJSONContent('One\nTwo')).toEqual({
type: 'doc',
content: [
{type: 'paragraph', content: [{type: 'text', text: 'One'}]},
{type: 'paragraph', content: [{type: 'text', text: 'Two'}]},
],
})
})
it('keeps single newlines as hard breaks once blank lines establish the paragraphing', () => {
expect(textToJSONContent('One\nTwo\n\nThree')).toEqual({
type: 'doc',
content: [
{
type: 'paragraph',
content: [{type: 'text', text: 'One'}, {type: 'hardBreak'}, {type: 'text', text: 'Two'}],
},
{type: 'paragraph', content: [{type: 'text', text: 'Three'}]},
],
})
})
it('collapses runs of blank lines and trims whitespace', () => {
expect(textToJSONContent(' A \n\n \n\n B ')).toEqual({
type: 'doc',
content: [
{type: 'paragraph', content: [{type: 'text', text: 'A'}]},
{type: 'paragraph', content: [{type: 'text', text: 'B'}]},
],
})
})
it('returns a valid empty doc for empty input', () => {
expect(textToJSONContent(' ')).toEqual({type: 'doc', content: [{type: 'paragraph'}]})
})
it('round-trips through the rich text serializer', () => {
const text = 'I grew up in Liège.\n\nI now live in Brussels.'
expect(parseJsonContentToText(textToJSONContent(text))).toBe(text)
})
})
describe('concatJSONContent', () => {
const doc = (...texts: string[]) => ({
type: 'doc',
content: texts.map((text) => ({type: 'paragraph', content: [{type: 'text', text}]})),
})
it('appends the second document after the first', () => {
expect(concatJSONContent(doc('One'), doc('Two'))).toEqual(doc('One', 'Two'))
})
it('does not accumulate blank paragraphs at the join', () => {
const trailingBlank = {
type: 'doc',
content: [{type: 'paragraph', content: [{type: 'text', text: 'One'}]}, {type: 'paragraph'}],
}
expect(concatJSONContent(trailingBlank, doc('Two'))).toEqual(doc('One', 'Two'))
})
it('returns the other side when one is missing or empty', () => {
expect(concatJSONContent(null, doc('Only'))).toEqual(doc('Only'))
expect(concatJSONContent(doc('Only'), undefined)).toEqual(doc('Only'))
})
it('returns a valid empty doc when both sides are empty', () => {
expect(concatJSONContent(null, null)).toEqual({type: 'doc', content: [{type: 'paragraph'}]})
})
})

View File

@@ -18,6 +18,13 @@ import {updateProfile} from 'web/lib/api'
import {useT} from 'web/lib/locale'
import {track} from 'web/lib/service/analytics'
/**
* Restores paragraph spacing for bios. The shared editor styling zeroes `p` margins at this size,
* which suits single-paragraph comments and chat bubbles but runs a multi-paragraph bio together.
* Must be `!important`: it competes with `prose-p:my-0` on the very same element.
*/
export const BIO_PARAGRAPH_SPACING = 'prose-p:!my-3'
export function BioTips({onClick}: {onClick?: () => void}) {
const t = useT()
const tips = t(
@@ -170,6 +177,10 @@ export function BaseBio({defaultValue, onBlur, onEditor, onClickTips}: BaseBioPr
// extensions: [StarterKit],
max: MAX_DESCRIPTION_LENGTH,
defaultValue: defaultValue,
// `proseClass` zeroes paragraph margins at this size — right for comments and chat bubbles,
// which are usually a single paragraph, but it makes a multi-paragraph bio render as one
// undifferentiated wall of text. `!` because it has to beat `prose-p:my-0` on the same element.
className: BIO_PARAGRAPH_SPACING,
placeholder: t(
'profile.bio.placeholder',
"Tell us all the details about yourself — and what you're looking for!",

View File

@@ -11,7 +11,7 @@ import {Tooltip} from 'web/components/widgets/tooltip'
import {updateProfile} from 'web/lib/api'
import {useT} from 'web/lib/locale'
import {EditableBio} from './editable-bio'
import {BIO_PARAGRAPH_SPACING, EditableBio} from './editable-bio'
export function BioBlock(props: {
isCurrentUser: boolean
@@ -33,7 +33,10 @@ export function BioBlock(props: {
<Row className="w-full">
{!edit && profile.bio && (
<Col className="flex w-full flex-grow" data-testid="profile-bio">
<Content className="w-full" content={profile.bio as JSONContent} />
<Content
className={clsx('w-full', BIO_PARAGRAPH_SPACING)}
content={profile.bio as JSONContent}
/>
</Col>
)}
{edit && (

View File

@@ -45,7 +45,7 @@ export function LLMExtractSection({
<div className="guidance">
{t(
'profile.llm.extract.guidance',
'Heads up: we use Google AI to extract your info. As we pay for the service, Google should not use this content to improve their models — but we never know. Prefer to keep things private? Just fill the form manually — no AI involved.',
'Heads up: we use Google AI to extract your info. As we pay for the service, Google should not use this content to improve their models — but we never know. Prefer to keep things fully internal? Just fill the form manually — no AI involved.',
)}
</div>
<BaseTextEditor
@@ -66,6 +66,7 @@ export function LLMExtractSection({
</div>
)}
<Button
color="indigo"
onClick={onExtract}
disabled={isExtracting || !parsingEditor?.getJSON?.() || isSubmitting}
loading={isExtracting}

View File

@@ -1,5 +1,6 @@
import {InformationCircleIcon} from '@heroicons/react/24/outline'
import {CameraIcon, InformationCircleIcon} from '@heroicons/react/24/outline'
import * as Sentry from '@sentry/node'
import {JSONContent} from '@tiptap/core'
import {Editor} from '@tiptap/react'
import clsx from 'clsx'
import {
@@ -32,6 +33,7 @@ import {MultipleChoiceOptions} from 'common/profiles/multiple-choice'
import {Profile, ProfileWithoutUser} from 'common/profiles/profile'
import {BaseUser} from 'common/user'
import {removeNullOrUndefinedProps} from 'common/util/object'
import {concatJSONContent, parseJsonContentToText} from 'common/util/parse'
import {urlize} from 'common/util/string'
import {MINUTE_MS, sleep} from 'common/util/time'
import {invert, range} from 'lodash'
@@ -48,6 +50,7 @@ import {LLMExtractSection} from 'web/components/llm-extract-section'
import {MultiCheckbox} from 'web/components/multi-checkbox'
import {City, CityRow, profileToCity, useCitySearch} from 'web/components/search-location'
import {SocialLinksSection} from 'web/components/social-links-section'
import {VoiceAutofillSection} from 'web/components/voice-autofill-section'
import {Carousel} from 'web/components/widgets/carousel'
import {ChoicesToggleGroup} from 'web/components/widgets/choices-toggle-group'
import {Input} from 'web/components/widgets/input'
@@ -58,6 +61,8 @@ import {ChoiceMap, ChoiceSetter, useChoicesContext} from 'web/hooks/use-choices'
import {api} from 'web/lib/api'
import {useLocale, useT} from 'web/lib/locale'
import {track} from 'web/lib/service/analytics'
import {blobToBase64} from 'web/lib/util/blob'
import {scrollIntoViewCentered} from 'web/lib/util/scroll'
import {colClassName, labelClassName} from 'web/pages/signup'
import {AddPhotosWidget} from './widgets/add-photos'
@@ -155,21 +160,31 @@ export const OptionalProfileUserForm = (props: {
const resetBig5 = () => BIG5_KEYS.forEach((k) => setProfile(k, null))
const [isExtracting, setIsExtracting] = useState(false)
const [autofillMode, setAutofillMode] = useState<'voice' | 'link'>('voice')
const [parsingEditor, setParsingEditor] = useState<any>(null)
const [extractionProgress, setExtractionProgress] = useState(0)
const [extractionError, setExtractionError] = useState<string | null>(null)
// Auto-fill can populate every field on the page, which makes the profile *look* finished — but it
// can never supply a photo. Set on a successful extraction; the banner itself also checks that
// there is still no photo, so it disappears the moment one is added.
const [remindAboutPhotos, setRemindAboutPhotos] = useState(false)
const photosRef = useRef<HTMLDivElement>(null)
const hasPhotos = !!profile.photo_urls?.length || !!profile.pinned_url
const showPhotoReminder = remindAboutPhotos && !hasPhotos
const handleLLMExtract = async (): Promise<Partial<ProfileWithoutUser>> => {
const llmContent = parsingEditor?.getText?.() ?? ''
if (!llmContent) {
toast.error(t('profile.llm.extract.error_empty', 'Please enter content to extract from'))
return {}
}
const runExtraction = async (input: {
content?: string
url?: string
source: 'text' | 'url' | 'voice'
}): Promise<Partial<ProfileWithoutUser>> => {
setIsExtracting(true)
setExtractionProgress(0)
setExtractionError(null)
const startTime = Date.now()
setInterval(() => {
// Was leaking: the interval was never cleared, so every extraction left another timer behind
// driving the progress bar. Now that a user can extract several times in a row (record, review,
// fill, record again) that compounds, so it is cleared in `finally`.
const progressInterval = setInterval(() => {
const elapsed = (Date.now() - startTime) / 1000
if (elapsed < 20) {
setExtractionProgress((elapsed / 30) * 100)
@@ -177,11 +192,7 @@ export const OptionalProfileUserForm = (props: {
setExtractionProgress((2 / 3) * 100 + ((elapsed - 20) / 150) * 100)
}
}, 100)
const isInputUrl = isUrl(llmContent)
const payload = {
locale,
...(isInputUrl ? {url: urlize(llmContent).trim()} : {content: llmContent.trim()}),
}
const payload = {locale, ...input}
try {
let extractedProfile: Partial<ProfileWithoutUser> = {}
let status: string | undefined = 'pending'
@@ -234,7 +245,19 @@ export const OptionalProfileUserForm = (props: {
} else if (key === 'keywords') setKeywordsString((value as string[]).join(', '))
;(extractedProfile as Record<string, unknown>)[key] = value
}
if (!isInputUrl) extractedProfile.bio = parsingEditor?.getJSON?.()
// Pasted text *is* the bio, so it is stored verbatim. A URL's bio comes back from the fetched
// page, and a voice bio is written by the LLM — a raw speech transcript reads terribly.
if (input.source === 'text') extractedProfile.bio = parsingEditor?.getJSON?.()
// A follow-up is an addition, not a correction: someone who remembered one more
// thing to say should not lose everything they already had. Only the bio behaves this way —
// the structured fields still take the newest answer, since those are single-valued.
if (extractedProfile.bio) {
const existingBio = profile.bio as JSONContent | null | undefined
if (parseJsonContentToText(existingBio).trim()) {
extractedProfile.bio = concatJSONContent(existingBio, extractedProfile.bio as JSONContent)
}
}
debug({
text: parsingEditor?.getText?.(),
json: parsingEditor?.getJSON?.(),
@@ -251,7 +274,7 @@ export const OptionalProfileUserForm = (props: {
t('profile.llm.extract.success', 'Profile data extracted! Please review below.'),
)
// clearInterval(progressInterval)
setRemindAboutPhotos(true)
setExtractionProgress(100)
return extractedProfile
@@ -269,12 +292,49 @@ export const OptionalProfileUserForm = (props: {
extra: {payload}, // for the rest (nested, etc.)
})
} finally {
// clearInterval(progressInterval)
clearInterval(progressInterval)
setIsExtracting(false)
}
return {}
}
const handleLLMExtract = async (): Promise<Partial<ProfileWithoutUser>> => {
const llmContent = parsingEditor?.getText?.() ?? ''
if (!llmContent) {
toast.error(t('profile.llm.extract.error_empty', 'Please enter content to extract from'))
return {}
}
return isUrl(llmContent)
? runExtraction({url: urlize(llmContent).trim(), source: 'url'})
: runExtraction({content: llmContent.trim(), source: 'text'})
}
const handleTranscribe = async (blob: Blob): Promise<string | null> => {
setExtractionError(null)
try {
const audio = await blobToBase64(blob)
const {transcript} = await api('transcribe-audio', {
audio,
mimeType: blob.type || 'audio/webm',
locale,
})
return transcript
} catch (error) {
console.error(error)
setExtractionError(
t(
'profile.voice.error.transcription',
'We could not transcribe your recording. Your recording is still here — try again, or fill in your profile manually.',
),
)
Sentry.captureException(error, {
user,
extra: {mimeType: blob.type, size: blob.size},
})
return null
}
}
const errorToast = () => {
toast.error(t('profile.optional.error.invalid_fields', 'Some fields are incorrect...'))
}
@@ -282,6 +342,8 @@ export const OptionalProfileUserForm = (props: {
const handleSubmit = async () => {
let finalProfile = profile
// Not gated on the visible tab: the editor stays mounted either way, so text typed here is real
// content the user would not expect us to drop just because they switched tabs before saving.
if (parsingEditor?.getText?.()?.trim()) {
const extractedProfile = await handleLLMExtract()
finalProfile = {...profile, ...extractedProfile}
@@ -399,19 +461,87 @@ export const OptionalProfileUserForm = (props: {
</p>
</div>
<Category title={t('profile.llm.extract.title', 'Auto-fill')} className={'mt-0'} />
<LLMExtractSection
parsingEditor={parsingEditor}
setParsingEditor={setParsingEditor}
isExtracting={isExtracting}
isSubmitting={isSubmitting}
onExtract={handleLLMExtract}
progress={extractionProgress}
/>
{/* Two ways in, one at a time: showing both a recorder and a text editor at once reads as
two things to do rather than a choice between them. */}
<Row className="w-full gap-1 rounded-xl bg-canvas-100 p-1 ring-1 ring-canvas-200">
{(
[
['voice', t('profile.autofill.by_voice', 'By voice')],
['link', t('profile.autofill.by_link', 'By link or text')],
] as const
).map(([mode, label]) => (
<button
key={mode}
type="button"
onClick={() => setAutofillMode(mode)}
aria-pressed={autofillMode === mode}
disabled={isExtracting}
className={clsx(
'flex-1 rounded-lg px-3 py-2 text-sm font-medium transition-colors disabled:cursor-not-allowed',
autofillMode === mode
? 'bg-canvas-0 text-ink-900 shadow-sm'
: 'text-ink-700 hover:text-ink-900',
)}
>
{label}
</button>
))}
</Row>
{/* Both panels stay mounted and the inactive one is hidden, rather than swapped out.
Unmounting the recorder would tear down an in-progress recording and drop any transcript
corrections the moment someone peeked at the other tab. `display: none` also keeps the
hidden panel out of the flex layout and away from assistive tech. */}
<div className={clsx(autofillMode !== 'voice' && 'hidden')}>
<VoiceAutofillSection
onTranscribe={handleTranscribe}
onExtract={async (transcript) => {
await runExtraction({content: transcript, source: 'voice'})
}}
isExtracting={isExtracting}
isSubmitting={isSubmitting}
progress={extractionProgress}
/>
</div>
<div className={clsx(autofillMode !== 'link' && 'hidden')}>
<LLMExtractSection
parsingEditor={parsingEditor}
setParsingEditor={setParsingEditor}
isExtracting={isExtracting}
isSubmitting={isSubmitting}
onExtract={handleLLMExtract}
progress={extractionProgress}
/>
</div>
{extractionError && (
<p className="border rounded-xl border-red-900 text-red-600 text-sm p-2">
{extractionError}
</p>
)}
{showPhotoReminder && (
<div
role="status"
className="flex items-start gap-3 rounded-xl bg-primary-100/60 ring-1 ring-primary-200 p-4"
>
<CameraIcon className="w-5 h-5 text-primary-700 shrink-0 mt-0.5" strokeWidth={1.8} />
<Col className="gap-2">
<p className="text-sm text-ink-700 leading-relaxed">
{t(
'profile.optional.photo_reminder',
'One thing we cannot fill in for you: your photos. They are still an important thing people look at (at least just to visually identify you), so do not leave without adding one or two.',
)}
</p>
<button
type="button"
className="text-primary-700 self-start text-sm font-medium hover:underline"
onClick={() => {
if (photosRef.current) scrollIntoViewCentered(photosRef.current)
}}
>
{t('profile.optional.photo_reminder.cta', 'Take me to photos')}
</button>
</Col>
</div>
)}
{/* The rule that used to sit here (`border border-b`, which draws two) is now owned by
`Category` itself, so every section boundary is drawn the same way. */}
<Category
@@ -636,7 +766,7 @@ export const OptionalProfileUserForm = (props: {
)}
</Col>
<Col className={clsx(colClassName)}>
<Col className={clsx(colClassName)} ref={photosRef}>
<label className={clsx(labelClassName)}>{t('profile.optional.photos', 'Photos')}</label>
{/*<div className="mb-1">*/}

View File

@@ -0,0 +1,348 @@
import {
ArrowPathIcon,
MicrophoneIcon,
PauseIcon,
PlayIcon,
StopIcon,
} from '@heroicons/react/24/solid'
import clsx from 'clsx'
import {useEffect, useState} from 'react'
import Textarea from 'react-expanding-textarea'
import {Button} from 'web/components/buttons/button'
import {Col} from 'web/components/layout/col'
import {Row} from 'web/components/layout/row'
import {AudioPlayer} from 'web/components/widgets/audio-player'
import {useAudioRecorder} from 'web/hooks/use-audio-recorder'
import {useT} from 'web/lib/locale'
import {
clearPendingRecording,
loadPendingRecording,
savePendingRecording,
savePendingTranscript,
} from 'web/lib/util/recording-store'
function formatTime(totalSeconds: number) {
const minutes = Math.floor(totalSeconds / 60)
const seconds = totalSeconds % 60
return `${minutes}:${String(seconds).padStart(2, '0')}`
}
/**
* Talking prompts. Deliberately open-ended and unordered — the point is to unblock someone staring
* at a mic button, not to turn speaking into another form to fill in.
*/
function useTalkingPoints() {
const t = useT()
return [
t('profile.voice.prompt.who', "What's your gender, how old are you, and where do you live?"),
t('profile.voice.prompt.work', 'What do you do — work, studies, projects?'),
t('profile.voice.prompt.time', 'How do you like to spend your time?'),
t('profile.voice.prompt.care', 'What do you care about most?'),
t('profile.voice.prompt.looking', 'What kind of connection are you looking for?'),
]
}
export function VoiceAutofillSection(props: {
/** Runs transcription; resolves to the transcript, or null when it failed. */
onTranscribe: (blob: Blob) => Promise<string | null>
/** Hands the (possibly edited) transcript to the profile extractor. */
onExtract: (transcript: string) => Promise<void>
isExtracting: boolean
isSubmitting: boolean
progress: number
}) {
const {onTranscribe, onExtract, isExtracting, isSubmitting, progress} = props
const t = useT()
const talkingPoints = useTalkingPoints()
const recorder = useAudioRecorder({
onRecorded: (recorded, recordedSeconds) => {
void savePendingRecording({
blob: recorded,
mimeType: recorded.type,
seconds: recordedSeconds,
transcript: null,
})
},
})
const {isRecording, isPaused, seconds, level, blob, previewUrl, restore} = recorder
const [isTranscribing, setIsTranscribing] = useState(false)
// Kept in state (not just handed straight to the extractor) so the user can read what we heard and
// fix any mis-transcribed names or places before it becomes their profile.
const [transcript, setTranscript] = useState<string | null>(null)
// Rehydrate whatever was left behind by a reload, an app close, or an earlier session. Runs once;
// `restore` no-ops if a recording is already under way.
useEffect(() => {
let cancelled = false
loadPendingRecording().then((pending) => {
if (cancelled || !pending) return
restore(pending.blob, pending.seconds)
if (pending.transcript) setTranscript(pending.transcript)
})
return () => {
cancelled = true
}
}, [restore])
const busy = isTranscribing || isExtracting || isSubmitting
const handleTranscribe = async () => {
if (!blob) return
setIsTranscribing(true)
try {
const text = await onTranscribe(blob)
if (text) {
setTranscript(text)
// Saved alongside the audio so a reload does not force a second (paid) transcription.
void savePendingTranscript(text)
}
} finally {
setIsTranscribing(false)
}
}
const handleExtract = async () => {
if (!transcript?.trim()) return
await onExtract(transcript.trim())
// The recording has served its purpose; the extracted fields are the artefact now.
setTranscript(null)
recorder.reset()
void clearPendingRecording()
}
const startOver = () => {
setTranscript(null)
recorder.reset()
void clearPendingRecording()
}
const errorMessage =
recorder.error === 'permission'
? t(
'profile.voice.error.permission',
'We could not access your microphone. Allow microphone access in your browser, then try again.',
)
: recorder.error === 'unsupported'
? t(
'profile.voice.error.unsupported',
'Your browser does not support voice recording. Try the link or text option instead.',
)
: recorder.error
? t('profile.voice.error.failed', 'Recording failed. Please try again.')
: null
return (
<Col className="gap-4">
<div>
{t(
'profile.voice.description',
'Rather talk than type? Record yourself for a few minutes and we will fill in your profile from what you say.',
)}
</div>
<div className="guidance">
{t(
'profile.voice.guidance',
'Heads up: your recording is sent to OpenAI Whisper to be transcribed, and the transcript to Google AI to fill in the fields. We pay for both services, so neither should use your content to train their models — but we never know. Prefer to keep things fully internal? Just fill the form manually — no AI involved.',
)}
</div>
{transcript === null ? (
<>
{/* Prompts and mic sit side by side from `sm` up: stacked, they pushed the mic below the
fold on short viewports. Talking points stay visible while recording — they are the
script. On mobile they stack, mic underneath. */}
<Col className="gap-4 sm:flex-row sm:items-center">
<div className="flex-1 rounded-xl bg-canvas-50 ring-1 ring-canvas-200 p-4">
<p className="text-sm font-semibold text-ink-900">
{t('profile.voice.prompts.title', 'Talk freely, or use these as a starting point:')}
</p>
<ul className="mt-2 space-y-1 text-sm text-ink-700">
{talkingPoints.map((point) => (
<li key={point} className="flex gap-2">
<span aria-hidden className="text-primary-500">
</span>
<span>{point}</span>
</li>
))}
</ul>
<p className="mt-3 text-xs text-ink-600">
{t(
'profile.voice.prompts.footer',
'No need to be polished — we clean up the ums and false starts. Skip anything you would rather not share.',
)}
</p>
</div>
{!blob && (
<Col className="shrink-0 items-center gap-2 sm:w-48">
<div className="relative flex h-28 w-28 items-center justify-center">
{isRecording && !isPaused && (
<>
<span
aria-hidden
className="absolute h-20 w-20 rounded-full bg-primary-500/15 will-change-transform"
style={{
transform: `scale(${1 + level * 1.6})`,
opacity: Math.max(0.05, 0.4 - level * 0.25),
transition: 'transform 140ms ease-out, opacity 140ms ease-out',
}}
/>
<span
aria-hidden
className="absolute h-20 w-20 rounded-full bg-primary-500/25 will-change-transform"
style={{
transform: `scale(${1 + level * 0.85})`,
transition: 'transform 140ms ease-out',
}}
/>
</>
)}
<button
type="button"
onClick={isRecording ? recorder.stop : recorder.start}
disabled={busy}
aria-label={
isRecording
? t('profile.voice.stop', 'Stop recording')
: t('profile.voice.start', 'Start recording')
}
className={clsx(
'relative flex h-20 w-20 items-center justify-center rounded-full text-white shadow-lg transition-transform active:scale-95 disabled:opacity-50',
// `bg-cta`, not `bg-primary-500`: the design system reserves this token for
// solid brand-coloured controls because white on primary-500 is only 3.30:1.
'bg-cta hover:bg-cta-hover',
)}
>
{isRecording ? (
<StopIcon className="h-8 w-8" />
) : (
<MicrophoneIcon className="h-9 w-9" />
)}
</button>
</div>
{isRecording ? (
<>
<span className="font-mono text-lg tabular-nums text-ink-900">
{formatTime(seconds)}
</span>
<span
className={clsx(
'text-sm',
isPaused ? 'text-ink-600' : 'animate-pulse font-medium text-primary-700',
)}
>
{isPaused
? t('profile.voice.paused', 'Paused')
: t('profile.voice.recording', 'Recording…')}
</span>
{/* Wraps rather than overflowing the narrow right-hand column. */}
<Row className="flex-wrap justify-center gap-2">
<Button
color="gray-outline"
size="sm"
onClick={isPaused ? recorder.resume : recorder.pause}
>
{isPaused ? (
<PlayIcon className="mr-1.5 h-4 w-4" />
) : (
<PauseIcon className="mr-1.5 h-4 w-4" />
)}
{isPaused
? t('profile.voice.resume', 'Resume')
: t('profile.voice.pause', 'Pause')}
</Button>
<Button color="indigo" size="sm" onClick={recorder.stop}>
{t('profile.voice.done_talking', 'I am done')}
</Button>
</Row>
</>
) : (
<span className="text-center text-sm text-ink-600">
{t('profile.voice.tap_to_record', 'Tap to start recording')}
</span>
)}
</Col>
)}
</Col>
{blob && !isRecording && (
<Col className="gap-3">
<p className="text-sm text-ink-700">
{t(
'profile.voice.review_recording',
'Feel free to listen before we transcribe it — {duration} recorded.',
).replace('{duration}', formatTime(seconds))}
</p>
{previewUrl && <AudioPlayer key={previewUrl} src={previewUrl} />}
<Row className="flex-wrap gap-2">
<Button
color="indigo"
onClick={handleTranscribe}
loading={isTranscribing}
disabled={busy}
>
{isTranscribing
? t('profile.voice.transcribing', 'Transcribing…')
: t('profile.voice.transcribe', 'Transcribe my recording')}
</Button>
<Button color="gray-outline" onClick={startOver} disabled={busy}>
<ArrowPathIcon className="mr-1.5 h-4 w-4" />
{t('profile.voice.record_again', 'Record again')}
</Button>
</Row>
</Col>
)}
</>
) : (
<Col className="gap-3">
<p className="text-sm text-ink-700">
{t(
'profile.voice.review_transcript',
'Here is what we heard. No need to tidy it up — the ums, the false starts, the times you changed your mind mid-sentence are all handled for you, and none of it ends up on your profile. Only worth fixing is what we misheard: names and places are the usual suspects.',
)}
</p>
<Textarea
value={transcript}
onChange={(e) => setTranscript(e.target.value)}
// Corrections to the transcript are persisted too, so a reload does not silently undo
// the name or city the user just fixed by hand.
onBlur={() => void savePendingTranscript(transcript)}
disabled={busy}
className="border-canvas-200 bg-canvas-0 text-ink-900 focus:border-primary-500 w-full resize-none rounded-lg border p-3 text-sm leading-relaxed focus:outline-none"
/>
{isExtracting && (
<div className="w-full h-2 bg-canvas-200 rounded-full overflow-hidden">
<div
className="h-full bg-primary-500 transition-all duration-100 ease-linear rounded-full"
style={{width: `${Math.min(progress, 100)}%`}}
/>
</div>
)}
<Row className="flex-wrap gap-2">
<Button
color="indigo"
onClick={handleExtract}
loading={isExtracting}
disabled={busy || !transcript.trim()}
>
{isExtracting
? t('profile.llm.extract.button_extracting', 'Extracting Profile Data')
: t('profile.voice.fill_profile', 'Fill in my profile')}
</Button>
<Button color="gray-outline" onClick={startOver} disabled={busy}>
<ArrowPathIcon className="mr-1.5 h-4 w-4" />
{t('profile.voice.record_again', 'Record again')}
</Button>
</Row>
</Col>
)}
{errorMessage && (
<p className="border rounded-xl border-red-900 text-red-600 text-sm p-2">{errorMessage}</p>
)}
</Col>
)
}

View File

@@ -0,0 +1,274 @@
import {PauseIcon, PlayIcon} from '@heroicons/react/24/solid'
import clsx from 'clsx'
import {debug} from 'common/logger'
import {useCallback, useEffect, useRef, useState} from 'react'
import {useT} from 'web/lib/locale'
function formatTime(totalSeconds: number) {
const safe = Number.isFinite(totalSeconds) && totalSeconds > 0 ? totalSeconds : 0
const minutes = Math.floor(safe / 60)
const seconds = Math.floor(safe % 60)
return `${minutes}:${String(seconds).padStart(2, '0')}`
}
/**
* Audio player driven by the Web Audio API rather than `<audio controls>`.
*
* Files produced by MediaRecorder (webm/opus) carry no duration in their container and are not
* seekable, so a native `<audio>` element reports `duration = Infinity`, paints its scrubber as
* full the instant playback starts, and refuses to move `currentTime`. Decoding the blob into an
* AudioBuffer up front gives us a real duration and lets us start playback from any offset — which
* is what makes both the moving playhead and dragging to seek possible. It also means the bar is
* ours to style, so it follows the app's palette instead of the browser's blue.
*/
export function AudioPlayer(props: {src: string; className?: string}) {
const {src, className} = props
const t = useT()
const trackRef = useRef<HTMLDivElement | null>(null)
const ctxRef = useRef<AudioContext | null>(null)
const bufferRef = useRef<AudioBuffer | null>(null)
const sourceRef = useRef<AudioBufferSourceNode | null>(null)
// Position (in seconds) at which the current playback segment started...
const offsetRef = useRef(0)
// ...and the context clock reading when it did, so elapsed time is a subtraction.
const startedAtRef = useRef(0)
const rafRef = useRef<number | null>(null)
const [ready, setReady] = useState(false)
const [playing, setPlaying] = useState(false)
const [current, setCurrent] = useState(0)
const [duration, setDuration] = useState(0)
const [dragging, setDragging] = useState(false)
useEffect(() => {
let cancelled = false
const Ctx =
window.AudioContext ||
(window as unknown as {webkitAudioContext: typeof AudioContext}).webkitAudioContext
const ctx = new Ctx()
ctxRef.current = ctx
offsetRef.current = 0
setReady(false)
setPlaying(false)
setCurrent(0)
fetch(src)
.then((r) => r.arrayBuffer())
.then((buf) => ctx.decodeAudioData(buf))
.then((decoded) => {
if (cancelled) return
bufferRef.current = decoded
setDuration(decoded.duration)
setReady(true)
})
.catch((e) => debug('failed to decode audio', e))
return () => {
cancelled = true
if (rafRef.current != null) cancelAnimationFrame(rafRef.current)
const source = sourceRef.current
if (source) {
source.onended = null
try {
source.stop()
} catch {
// Already stopped; nothing to do.
}
source.disconnect()
}
sourceRef.current = null
bufferRef.current = null
ctx.close().catch(() => {})
}
}, [src])
const currentPosition = useCallback(() => {
const ctx = ctxRef.current
const buffer = bufferRef.current
if (!ctx || !buffer || !sourceRef.current) return offsetRef.current
return Math.min(buffer.duration, offsetRef.current + (ctx.currentTime - startedAtRef.current))
}, [])
const stopSource = useCallback(() => {
const source = sourceRef.current
if (source) {
// Manual stop: drop the handler so it is not mistaken for reaching the end.
source.onended = null
try {
source.stop()
} catch {
// Already stopped.
}
try {
source.disconnect()
} catch {
// Already disconnected.
}
sourceRef.current = null
}
if (rafRef.current != null) {
cancelAnimationFrame(rafRef.current)
rafRef.current = null
}
}, [])
const startSegment = useCallback(
(offset: number) => {
const ctx = ctxRef.current
const buffer = bufferRef.current
if (!ctx || !buffer) return
stopSource()
const node = ctx.createBufferSource()
node.buffer = buffer
node.connect(ctx.destination)
node.onended = () => {
// Played to the end: rewind so the button is a straightforward replay.
sourceRef.current = null
offsetRef.current = 0
setPlaying(false)
setCurrent(0)
}
offsetRef.current = Math.min(Math.max(0, offset), buffer.duration)
startedAtRef.current = ctx.currentTime
node.start(0, offsetRef.current)
sourceRef.current = node
const tick = () => {
const buf = bufferRef.current
if (!buf) return
const pos = currentPosition()
setCurrent(pos)
if (sourceRef.current && pos < buf.duration) rafRef.current = requestAnimationFrame(tick)
}
rafRef.current = requestAnimationFrame(tick)
},
[currentPosition, stopSource],
)
const play = useCallback(() => {
if (!ctxRef.current || !bufferRef.current) return
// Browsers start the context suspended until a user gesture.
ctxRef.current.resume().catch(() => {})
startSegment(offsetRef.current)
setPlaying(true)
}, [startSegment])
const pause = useCallback(() => {
offsetRef.current = currentPosition()
stopSource()
setPlaying(false)
setCurrent(offsetRef.current)
}, [currentPosition, stopSource])
const seekTo = useCallback(
(seconds: number) => {
const buffer = bufferRef.current
if (!buffer) return
const clamped = Math.min(Math.max(0, seconds), buffer.duration)
offsetRef.current = clamped
setCurrent(clamped)
if (playing) startSegment(clamped)
},
[playing, startSegment],
)
const seekToPointer = (clientX: number) => {
const buffer = bufferRef.current
const element = trackRef.current
if (!buffer || !element) return
const rect = element.getBoundingClientRect()
const ratio = Math.min(1, Math.max(0, (clientX - rect.left) / rect.width))
seekTo(ratio * buffer.duration)
}
const endDrag = (e: React.PointerEvent<HTMLDivElement>) => {
if (e.currentTarget.hasPointerCapture(e.pointerId)) {
e.currentTarget.releasePointerCapture(e.pointerId)
}
setDragging(false)
}
const percent = duration ? Math.min(100, (current / duration) * 100) : 0
return (
<div
className={clsx(
'flex w-full items-center gap-3 rounded-xl bg-canvas-50 ring-1 ring-canvas-200 px-3 py-2.5',
className,
)}
>
<button
type="button"
onClick={playing ? pause : play}
disabled={!ready}
aria-label={playing ? t('audio.pause', 'Pause') : t('audio.play', 'Play')}
// `bg-cta` is the design system's solid brand fill; white on `primary-500` is only 3.30:1.
className="bg-cta hover:bg-cta-hover flex h-10 w-10 shrink-0 items-center justify-center rounded-full text-white transition-colors active:scale-95 disabled:opacity-50"
>
{playing ? <PauseIcon className="h-5 w-5" /> : <PlayIcon className="h-5 w-5 pl-0.5" />}
</button>
<div className="min-w-0 flex-1">
{/* Negative margin plus padding: a comfortably tall hit area around a slim visual bar. */}
<div
ref={trackRef}
onPointerDown={(e) => {
if (!ready) return
e.currentTarget.setPointerCapture(e.pointerId)
setDragging(true)
seekToPointer(e.clientX)
}}
onPointerMove={(e) => {
if (dragging) seekToPointer(e.clientX)
}}
onPointerUp={endDrag}
onPointerCancel={endDrag}
onKeyDown={(e) => {
if (!ready) return
if (e.key === 'ArrowLeft') {
e.preventDefault()
seekTo(currentPosition() - 5)
} else if (e.key === 'ArrowRight') {
e.preventDefault()
seekTo(currentPosition() + 5)
} else if (e.key === 'Home') {
e.preventDefault()
seekTo(0)
} else if (e.key === 'End') {
e.preventDefault()
seekTo(duration)
}
}}
className="relative -my-2 cursor-pointer touch-none py-2"
role="slider"
tabIndex={0}
aria-label={t('audio.position', 'Playback position')}
aria-valuemin={0}
aria-valuemax={Math.round(duration)}
aria-valuenow={Math.round(current)}
aria-valuetext={formatTime(current)}
>
<div className="relative h-1.5 rounded-full bg-canvas-200">
<div
className="bg-primary-500 absolute inset-y-0 left-0 rounded-full"
style={{width: `${percent}%`}}
/>
<div
className={clsx(
'bg-primary-500 ring-canvas-0 absolute top-1/2 h-3.5 w-3.5 -translate-x-1/2 -translate-y-1/2 rounded-full shadow ring-2 transition-transform',
dragging && 'scale-110',
)}
style={{left: `${percent}%`}}
/>
</div>
</div>
<div className="mt-1 flex justify-between font-mono text-[11px] tabular-nums text-ink-600">
<span>{formatTime(current)}</span>
<span>{duration ? formatTime(duration) : '—:—'}</span>
</div>
</div>
</div>
)
}

View File

@@ -0,0 +1,244 @@
import {debug} from 'common/logger'
import {useCallback, useEffect, useMemo, useRef, useState} from 'react'
/**
* Ordered by preference. Opus in WebM is what Chromium records; `audio/mp4` (AAC) is the only thing
* iOS Safari will give us. Both are accepted by the transcription backend.
*/
const MIME_CANDIDATES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/aac']
// Speech does not benefit from more, and every extra kbps is base64 we have to push over the wire.
const AUDIO_BITS_PER_SECOND = 32000
function pickMimeType(): string | undefined {
if (typeof MediaRecorder === 'undefined' || !MediaRecorder.isTypeSupported) return undefined
return MIME_CANDIDATES.find((type) => MediaRecorder.isTypeSupported(type))
}
export type RecorderError = 'permission' | 'unsupported' | 'failed'
/**
* Microphone recording with a live input level for visual feedback.
*
* The blob is kept until the caller explicitly clears it, so a failed upload never costs the user
* their recording — they can retry or re-listen.
*/
export function useAudioRecorder(
opts: {
/**
* Fired once a recording is finalised, before anything is uploaded — the hook's cue to the
* caller to persist it. Kept in a ref so passing an inline closure does not re-create `start`.
*/
onRecorded?: (blob: Blob, seconds: number) => void
} = {},
) {
const onRecordedRef = useRef(opts.onRecorded)
onRecordedRef.current = opts.onRecorded
const [isRecording, setIsRecording] = useState(false)
const [isPaused, setIsPaused] = useState(false)
const [seconds, setSeconds] = useState(0)
// Smoothed microphone amplitude, 0..1. Drives the pulsing rings.
const [level, setLevel] = useState(0)
const [blob, setBlob] = useState<Blob | null>(null)
const [error, setError] = useState<RecorderError | null>(null)
const recorderRef = useRef<MediaRecorder | null>(null)
const chunksRef = useRef<BlobPart[]>([])
const streamRef = useRef<MediaStream | null>(null)
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null)
const audioCtxRef = useRef<AudioContext | null>(null)
const rafRef = useRef<number | null>(null)
const levelRef = useRef(0)
// `onstop` fires outside React's render cycle, so it cannot read `seconds` from the closure.
const secondsRef = useRef(0)
useEffect(() => {
secondsRef.current = seconds
}, [seconds])
const previewUrl = useMemo(() => (blob ? URL.createObjectURL(blob) : null), [blob])
useEffect(() => {
return () => {
if (previewUrl) URL.revokeObjectURL(previewUrl)
}
}, [previewUrl])
const stopTimer = useCallback(() => {
if (timerRef.current) clearInterval(timerRef.current)
timerRef.current = null
}, [])
const startTimer = useCallback(() => {
stopTimer()
timerRef.current = setInterval(() => setSeconds((s) => s + 1), 1000)
}, [stopTimer])
const stopMeter = useCallback(() => {
if (rafRef.current != null) cancelAnimationFrame(rafRef.current)
rafRef.current = null
audioCtxRef.current?.close().catch(() => {})
audioCtxRef.current = null
levelRef.current = 0
setLevel(0)
}, [])
const startMeter = useCallback((stream: MediaStream) => {
try {
const Ctx =
window.AudioContext ||
(window as unknown as {webkitAudioContext: typeof AudioContext}).webkitAudioContext
const ctx = new Ctx()
audioCtxRef.current = ctx
const analyser = ctx.createAnalyser()
analyser.fftSize = 512
ctx.createMediaStreamSource(stream).connect(analyser)
const data = new Uint8Array(analyser.fftSize)
levelRef.current = 0
const tick = () => {
analyser.getByteTimeDomainData(data)
let sum = 0
for (let i = 0; i < data.length; i++) {
const v = (data[i] - 128) / 128
sum += v * v
}
const rms = Math.sqrt(sum / data.length)
// High gain plus a sub-linear curve, so a normal speaking voice (RMS ~0.05) already fills
// most of the visible range instead of barely moving.
const target = Math.pow(Math.min(1, rms * 6.5), 0.6)
// Asymmetric smoothing: quick to rise, slow to fall, so the rings breathe rather than
// snapping on every syllable.
const prev = levelRef.current
const next = prev + (target - prev) * (target > prev ? 0.28 : 0.08)
levelRef.current = next
setLevel(next)
rafRef.current = requestAnimationFrame(tick)
}
tick()
} catch (e) {
// Purely decorative — never let it break the recording itself.
debug('audio level meter unavailable', e)
}
}, [])
// Teardown on unmount: an orphaned MediaStream keeps the browser's recording indicator lit.
useEffect(() => {
return () => {
stopTimer()
stopMeter()
if (recorderRef.current && recorderRef.current.state !== 'inactive') {
recorderRef.current.stop()
}
streamRef.current?.getTracks().forEach((t) => t.stop())
}
}, [stopMeter, stopTimer])
const start = useCallback(async () => {
setError(null)
if (typeof MediaRecorder === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
setError('unsupported')
return
}
try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: {echoCancellation: true, noiseSuppression: true},
})
streamRef.current = stream
const mimeType = pickMimeType()
const recorder = new MediaRecorder(stream, {
...(mimeType ? {mimeType} : {}),
audioBitsPerSecond: AUDIO_BITS_PER_SECOND,
})
chunksRef.current = []
recorder.ondataavailable = (e) => {
if (e.data.size > 0) chunksRef.current.push(e.data)
}
recorder.onstop = () => {
stream.getTracks().forEach((t) => t.stop())
streamRef.current = null
const type = recorder.mimeType || mimeType || 'audio/webm'
const recorded = new Blob(chunksRef.current, {type})
setBlob(recorded)
// Persist before any network call, so a crash mid-upload still leaves the audio on disk.
onRecordedRef.current?.(recorded, secondsRef.current)
}
recorder.start()
recorderRef.current = recorder
setBlob(null)
setSeconds(0)
setIsRecording(true)
setIsPaused(false)
startTimer()
startMeter(stream)
} catch (e) {
debug('microphone unavailable', e)
const name = (e as {name?: string})?.name
setError(name === 'NotAllowedError' || name === 'SecurityError' ? 'permission' : 'failed')
streamRef.current?.getTracks().forEach((t) => t.stop())
streamRef.current = null
}
}, [startMeter, startTimer])
const pause = useCallback(() => {
const recorder = recorderRef.current
if (recorder?.state !== 'recording') return
recorder.pause()
setIsPaused(true)
stopTimer()
stopMeter()
}, [stopMeter, stopTimer])
const resume = useCallback(() => {
const recorder = recorderRef.current
if (recorder?.state !== 'paused') return
recorder.resume()
setIsPaused(false)
startTimer()
if (streamRef.current) startMeter(streamRef.current)
}, [startMeter, startTimer])
const stop = useCallback(() => {
if (recorderRef.current && recorderRef.current.state !== 'inactive') {
recorderRef.current.stop()
}
setIsRecording(false)
setIsPaused(false)
stopTimer()
stopMeter()
}, [stopMeter, stopTimer])
/**
* Adopts a previously saved recording as the current one — used to rehydrate from storage after a
* reload. Ignored while recording, so a slow restore can never clobber live audio.
*/
const restore = useCallback((restored: Blob, recordedSeconds: number) => {
if (recorderRef.current && recorderRef.current.state !== 'inactive') return
setBlob(restored)
setSeconds(recordedSeconds)
}, [])
/** Throws away the recording and returns to the initial state. */
const reset = useCallback(() => {
stop()
setBlob(null)
setSeconds(0)
setError(null)
}, [stop])
return {
isRecording,
isPaused,
seconds,
level,
blob,
previewUrl,
error,
start,
pause,
restore,
resume,
stop,
reset,
}
}

22
web/lib/util/blob.ts Normal file
View File

@@ -0,0 +1,22 @@
/**
* Base64-encodes a Blob, without the `data:...;base64,` prefix.
*
* FileReader rather than `Buffer`/`btoa`: it streams, so a multi-megabyte audio recording does not
* have to be materialised as a JS string of char codes first.
*/
export function blobToBase64(blob: Blob): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader()
reader.onerror = () => reject(reader.error ?? new Error('Failed to read blob'))
reader.onload = () => {
const result = reader.result
if (typeof result !== 'string') {
reject(new Error('Unexpected FileReader result'))
return
}
const comma = result.indexOf(',')
resolve(comma === -1 ? result : result.slice(comma + 1))
}
reader.readAsDataURL(blob)
})
}

View File

@@ -0,0 +1,109 @@
import {debug} from 'common/logger'
import {WEEK_MS} from 'common/util/time'
/**
* On-device persistence for an in-progress voice auto-fill recording, so that switching tabs,
* reloading, or closing the app never costs someone the minute they just spent talking.
*
* IndexedDB rather than `localStorage`, for three reasons:
* - it stores a `Blob` natively; localStorage is strings only, so audio would have to be base64'd,
* inflating it by a third
* - localStorage's ~5 MB origin budget is shared with everything else we keep there, and
* `safeLocalStorage` reacts to a quota error by calling `store.clear()` — a long recording could
* therefore wipe the user's saved profile draft
* - localStorage is synchronous, so writing megabytes of audio would jank the main thread
*/
const DB_NAME = 'compass'
const STORE = 'pending-recordings'
const VERSION = 1
// Only ever one profile recording in flight, so a fixed key keeps this a single-row store.
const KEY = 'profile-voice-autofill'
// Audio is bulky; don't sit on someone's voice indefinitely if they never came back to it.
const TTL_MS = WEEK_MS
export type PendingRecording = {
blob: Blob
mimeType: string
/** Recording length in seconds, so the UI can show it without decoding the audio. */
seconds: number
/** Present once transcribed, so a reload does not force a second (paid) transcription. */
transcript: string | null
createdAt: number
}
function openDb(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const req = indexedDB.open(DB_NAME, VERSION)
req.onupgradeneeded = () => {
const db = req.result
if (!db.objectStoreNames.contains(STORE)) db.createObjectStore(STORE)
}
req.onsuccess = () => resolve(req.result)
req.onerror = () => reject(req.error)
req.onblocked = () => reject(new Error('IndexedDB upgrade blocked'))
})
}
function tx<T>(mode: IDBTransactionMode, run: (store: IDBObjectStore) => IDBRequest<T>): Promise<T> {
return openDb().then(
(db) =>
new Promise<T>((resolve, reject) => {
const request = run(db.transaction(STORE, mode).objectStore(STORE))
request.onsuccess = () => resolve(request.result)
request.onerror = () => reject(request.error)
}),
)
}
// Private browsing, disabled storage and SSR all make IndexedDB unavailable. Persistence is a
// convenience, never a precondition for recording, so every failure degrades to "not saved".
const available = () => typeof indexedDB !== 'undefined'
export async function savePendingRecording(
entry: Omit<PendingRecording, 'createdAt'>,
): Promise<void> {
if (!available()) return
try {
await tx('readwrite', (s) => s.put({...entry, createdAt: Date.now()}, KEY))
} catch (e) {
debug('failed to persist recording', e)
}
}
/** Updates the transcript on the stored recording, leaving the audio untouched. */
export async function savePendingTranscript(transcript: string | null): Promise<void> {
if (!available()) return
try {
const existing = await tx<PendingRecording | undefined>('readonly', (s) => s.get(KEY))
if (!existing) return
await tx('readwrite', (s) => s.put({...existing, transcript}, KEY))
} catch (e) {
debug('failed to persist transcript', e)
}
}
export async function loadPendingRecording(): Promise<PendingRecording | null> {
if (!available()) return null
try {
const entry = await tx<PendingRecording | undefined>('readonly', (s) => s.get(KEY))
if (!entry) return null
if (Date.now() - entry.createdAt > TTL_MS) {
await clearPendingRecording()
return null
}
return entry
} catch (e) {
debug('failed to load recording', e)
return null
}
}
export async function clearPendingRecording(): Promise<void> {
if (!available()) return
try {
await tx('readwrite', (s) => s.delete(KEY))
} catch (e) {
debug('failed to clear recording', e)
}
}