From 92500b4389a26aecb2101fb795f534fd3b039afb Mon Sep 17 00:00:00 2001 From: Stefan Walcz Date: Sun, 18 Oct 2026 21:52:59 +0200 Subject: [PATCH] feat(ui): dictate in the chat composer and read scanned PDFs as images (#12756) Two gaps in the chat composer: - The microphone button only opened a file dialog for audio files; there was no way to speak a message. Add a dictation button: the first click records from the microphone, the second stops and sends the recording to /v1/audio/transcriptions with the first installed transcription model; the text is appended to the message box for review. The button is hidden without a transcription model or without a microphone API. The audio attach button keeps its function and gets the music icon. - A PDF without a text layer (a scan) was rejected with a toast. Render its first ten pages to PNG with pdf.js and attach them as images, so a vision model can read them; a toast says how many pages went along. PDFs that cannot be parsed at all are still rejected. Strings for all eight locales, docs in features/chat.md, Playwright specs for both (with negative cases: no transcription model, failed transcription, unparsable PDF). Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Stefan Walcz --- core/http/react-ui/e2e/chat-dictation.spec.js | 74 +++++++++++++++ .../react-ui/e2e/chat-pdf-attachment.spec.js | 40 +++++++- .../http/react-ui/public/locales/de/chat.json | 3 +- .../http/react-ui/public/locales/de/home.json | 9 +- .../http/react-ui/public/locales/en/chat.json | 3 +- .../http/react-ui/public/locales/en/home.json | 9 +- .../http/react-ui/public/locales/es/chat.json | 3 +- .../http/react-ui/public/locales/es/home.json | 9 +- .../http/react-ui/public/locales/id/chat.json | 3 +- .../http/react-ui/public/locales/id/home.json | 9 +- .../http/react-ui/public/locales/it/chat.json | 3 +- .../http/react-ui/public/locales/it/home.json | 9 +- .../http/react-ui/public/locales/ko/chat.json | 3 +- .../http/react-ui/public/locales/ko/home.json | 9 +- .../react-ui/public/locales/pt-BR/chat.json | 3 +- .../react-ui/public/locales/pt-BR/home.json | 9 +- .../react-ui/public/locales/zh-CN/chat.json | 3 +- .../react-ui/public/locales/zh-CN/home.json | 9 +- .../src/components/home/DictateButton.jsx | 94 +++++++++++++++++++ .../src/components/home/HomeComposer.jsx | 10 +- core/http/react-ui/src/pages/Chat.jsx | 16 +++- core/http/react-ui/src/pages/Home.jsx | 16 +++- core/http/react-ui/src/utils/pdf.js | 40 +++++++- docs/content/features/chat.md | 4 + 24 files changed, 365 insertions(+), 25 deletions(-) create mode 100644 core/http/react-ui/e2e/chat-dictation.spec.js create mode 100644 core/http/react-ui/src/components/home/DictateButton.jsx diff --git a/core/http/react-ui/e2e/chat-dictation.spec.js b/core/http/react-ui/e2e/chat-dictation.spec.js new file mode 100644 index 000000000..cab8d53cd --- /dev/null +++ b/core/http/react-ui/e2e/chat-dictation.spec.js @@ -0,0 +1,74 @@ +import { test, expect } from './coverage-fixtures.js' +import { chatOf, mockChat, openChat, MODELS } from './chat-fixtures.js' + +// Dictation in the composer: record from the microphone, send the recording to +// /v1/audio/transcriptions with the installed transcription model, put the text +// into the message box. The browser has no microphone here, so getUserMedia and +// MediaRecorder are replaced with stand-ins that hand back a few bytes. + +const WHISPER = { id: 'whisper-1', capabilities: ['FLAG_TRANSCRIPT'] } + +async function fakeMicrophone(page) { + await page.addInitScript(() => { + navigator.mediaDevices.getUserMedia = async () => ({ getTracks: () => [{ stop() { window.__micStopped = true } }] }) + window.MediaRecorder = class { + constructor() { this.state = 'inactive'; this.mimeType = 'audio/webm' } + start() { this.state = 'recording' } + stop() { + this.state = 'inactive' + this.ondataavailable?.({ data: new Blob(['voice'], { type: 'audio/webm' }) }) + this.onstop?.() + } + } + }) +} + +const input = (page) => page.getByTestId('chat-input') + +test.describe('Chat dictation', () => { + test('records, transcribes with the transcription model and appends the text', async ({ page }) => { + await fakeMicrophone(page) + await mockChat(page, { models: [...MODELS, WHISPER] }) + let form = '' + await page.route('**/v1/audio/transcriptions', (route) => { + form = route.request().postDataBuffer()?.toString('latin1') || '' + route.fulfill({ json: { text: ' Wie spät ist es? ' } }) + }) + await openChat(page, [chatOf('c1', 'Dictate', 'qwen3-8b', [])]) + + await input(page).fill('Frage:') + const btn = page.getByTestId('chat-dictate') + await btn.click() + await expect(btn).toHaveAttribute('data-state', 'recording') + await expect(btn).toHaveAttribute('aria-pressed', 'true') + await btn.click() + + await expect(input(page)).toHaveValue('Frage: Wie spät ist es?') + await expect(btn).toHaveAttribute('data-state', 'idle') + expect(form).toContain('name="model"\r\n\r\nwhisper-1') + expect(form).toContain('filename="dictation.webm"') + expect(await page.evaluate(() => window.__micStopped)).toBe(true) + }) + + test('shows an error and keeps the message when transcription fails', async ({ page }) => { + await fakeMicrophone(page) + await mockChat(page, { models: [...MODELS, WHISPER] }) + await page.route('**/v1/audio/transcriptions', (route) => + route.fulfill({ status: 500, json: { error: { message: 'backend down' } } })) + await openChat(page, [chatOf('c1', 'Dictate', 'qwen3-8b', [])]) + + await input(page).fill('bleibt') + await page.getByTestId('chat-dictate').click() + await page.getByTestId('chat-dictate').click() + await expect(page.getByText(/Transcription failed/)).toBeVisible() + await expect(input(page)).toHaveValue('bleibt') + }) + + test('is not offered without a transcription model', async ({ page }) => { + await fakeMicrophone(page) + await mockChat(page) + await openChat(page, [chatOf('c1', 'Dictate', 'qwen3-8b', [])]) + await expect(page.getByRole('button', { name: 'Attach audio' })).toBeVisible() + await expect(page.getByTestId('chat-dictate')).toHaveCount(0) + }) +}) diff --git a/core/http/react-ui/e2e/chat-pdf-attachment.spec.js b/core/http/react-ui/e2e/chat-pdf-attachment.spec.js index 1bb6bdd23..09251d603 100644 --- a/core/http/react-ui/e2e/chat-pdf-attachment.spec.js +++ b/core/http/react-ui/e2e/chat-pdf-attachment.spec.js @@ -2,8 +2,8 @@ import { test, expect } from './coverage-fixtures.js' // Single-page PDF with a real text layer, built byte by byte so the xref // offsets are valid and the spec needs no binary fixture. -function buildPdf(text) { - const stream = `BT /F1 18 Tf 20 100 Td (${text}) Tj ET` +function buildPdf(text, { stream: rawStream } = {}) { + const stream = rawStream ?? `BT /F1 18 Tf 20 100 Td (${text}) Tj ET` const objs = [ '<< /Type /Catalog /Pages 2 0 R >>', '<< /Type /Pages /Kids [3 0 R] /Count 1 >>', @@ -73,6 +73,42 @@ test.describe('Chat - PDF attachments', () => { }) }) +// A page that only paints a filled rectangle: what a scan looks like to pdf.js +// (an image, no text layer). +const scannedPdf = () => buildPdf('', { stream: '0 0 1 rg 20 20 200 120 re f' }) + +test.describe('Chat - scanned PDF attachments', () => { + test('sends the pages of a PDF without text layer as images', async ({ page }) => { + let requestBody = '' + await page.route('**/v1/chat/completions', (route) => { + requestBody = route.request().postData() || '' + route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: { message: 'stop' } }) }) + }) + await openChat(page) + + await page.locator('input[type=file][accept*="pdf"]').setInputFiles({ + name: 'scan.pdf', mimeType: 'application/pdf', buffer: scannedPdf(), + }) + await expect(page.getByText('scan.pdf has no text layer')).toBeVisible({ timeout: 10_000 }) + await expect(page.locator('.home-file-tag', { hasText: 'scan-page-1.png' })).toBeVisible() + + await page.getByTestId('chat-input').fill('What does this say?') + await page.getByTestId('chat-send').click() + + await expect.poll(() => requestBody).toContain('data:image/png;base64,') + expect(requestBody).not.toContain('%PDF') + }) + + test('still rejects a PDF that cannot be parsed at all', async ({ page }) => { + await openChat(page) + await page.locator('input[type=file][accept*="pdf"]').setInputFiles({ + name: 'broken.pdf', mimeType: 'application/pdf', buffer: Buffer.from('%PDF-1.4 this is not a real document'), + }) + await expect(page.getByText('Could not read text from broken.pdf')).toBeVisible({ timeout: 10_000 }) + await expect(page.locator('.home-file-tag', { hasText: 'broken' })).toHaveCount(0) + }) +}) + test.describe('Home - PDF attachments', () => { test('attaches a PDF that has a text layer', async ({ page }) => { await page.goto('/app') diff --git a/core/http/react-ui/public/locales/de/chat.json b/core/http/react-ui/public/locales/de/chat.json index 76cd0557b..83963d5da 100644 --- a/core/http/react-ui/public/locales/de/chat.json +++ b/core/http/react-ui/public/locales/de/chat.json @@ -103,7 +103,8 @@ "toasts": { "selectModel": "Bitte wählen Sie ein Modell", "copied": "In die Zwischenablage kopiert", - "copyFailed": "Kopieren in die Zwischenablage fehlgeschlagen" + "copyFailed": "Kopieren in die Zwischenablage fehlgeschlagen", + "pdfScannedAsImages": "{{name}} hat keine Textebene (Scan). {{count}} von {{total}} Seiten als Bilder für ein Bildmodell angehängt." }, "menu": { "trigger": "Chats", diff --git a/core/http/react-ui/public/locales/de/home.json b/core/http/react-ui/public/locales/de/home.json index 80a063685..5ee65e007 100644 --- a/core/http/react-ui/public/locales/de/home.json +++ b/core/http/react-ui/public/locales/de/home.json @@ -35,7 +35,14 @@ "enterToSend": "Enter zum Senden", "selectModelFirst": "Bitte zuerst ein Modell auswählen", "sendMessage": "Nachricht senden", - "selectModelToast": "Bitte zuerst ein Modell auswählen" + "selectModelToast": "Bitte zuerst ein Modell auswählen", + "dictate": "Diktieren (Sprache zu Text)", + "dictateStop": "Aufnahme beenden", + "dictateBusy": "Wird umgewandelt…", + "dictateEmpty": "Keine Sprache erkannt.", + "dictateFailed": "Umwandlung fehlgeschlagen: {{error}}", + "dictateNoMic": "Mikrofon nicht verfügbar: {{error}}", + "pdfScannedAsImages": "{{name}} hat keine Textebene (Scan). {{count}} von {{total}} Seiten als Bilder für ein Bildmodell angehängt." }, "quickLinks": { "manageByChat": "Per Chat verwalten", diff --git a/core/http/react-ui/public/locales/en/chat.json b/core/http/react-ui/public/locales/en/chat.json index b714380ad..f6e428101 100644 --- a/core/http/react-ui/public/locales/en/chat.json +++ b/core/http/react-ui/public/locales/en/chat.json @@ -155,7 +155,8 @@ "copyFailed": "Could not copy to clipboard", "chatCopied": "Chat copied to clipboard", "forked": "Created a new chat", - "pdfReadFailed": "Could not read text from {{name}}. It may be scanned, encrypted or damaged." + "pdfReadFailed": "Could not read text from {{name}}. It may be scanned, encrypted or damaged.", + "pdfScannedAsImages": "{{name}} has no text layer (scan). Attached {{count}} of {{total}} pages as images for a vision model." }, "menu": { "trigger": "Chats", diff --git a/core/http/react-ui/public/locales/en/home.json b/core/http/react-ui/public/locales/en/home.json index a65c2ddaf..75a949802 100644 --- a/core/http/react-ui/public/locales/en/home.json +++ b/core/http/react-ui/public/locales/en/home.json @@ -44,7 +44,14 @@ "enterHint": "Enter to send, Shift+Enter for a new line", "send": "Send", "removeFile": "Remove {{name}}", - "stop": "Stop" + "stop": "Stop", + "dictate": "Dictate (speech to text)", + "dictateStop": "Stop recording", + "dictateBusy": "Transcribing…", + "dictateEmpty": "No speech recognised.", + "dictateFailed": "Transcription failed: {{error}}", + "dictateNoMic": "Microphone not available: {{error}}", + "pdfScannedAsImages": "{{name}} has no text layer (scan). Attached {{count}} of {{total}} pages as images for a vision model." }, "quickLinks": { "manageByChat": "Manage by chat", diff --git a/core/http/react-ui/public/locales/es/chat.json b/core/http/react-ui/public/locales/es/chat.json index 0b551f045..d8a73a553 100644 --- a/core/http/react-ui/public/locales/es/chat.json +++ b/core/http/react-ui/public/locales/es/chat.json @@ -103,7 +103,8 @@ "toasts": { "selectModel": "Por favor selecciona un modelo", "copied": "Copiado al portapapeles", - "copyFailed": "No se pudo copiar al portapapeles" + "copyFailed": "No se pudo copiar al portapapeles", + "pdfScannedAsImages": "{{name}} no tiene capa de texto (escaneo). Se adjuntaron {{count}} de {{total}} páginas como imágenes para un modelo de visión." }, "menu": { "trigger": "Chats", diff --git a/core/http/react-ui/public/locales/es/home.json b/core/http/react-ui/public/locales/es/home.json index cfa916826..8b9a9c807 100644 --- a/core/http/react-ui/public/locales/es/home.json +++ b/core/http/react-ui/public/locales/es/home.json @@ -35,7 +35,14 @@ "enterToSend": "Enter para enviar", "selectModelFirst": "Selecciona un modelo primero", "sendMessage": "Enviar mensaje", - "selectModelToast": "Por favor selecciona un modelo primero" + "selectModelToast": "Por favor selecciona un modelo primero", + "dictate": "Dictar (voz a texto)", + "dictateStop": "Detener grabación", + "dictateBusy": "Transcribiendo…", + "dictateEmpty": "No se reconoció ninguna voz.", + "dictateFailed": "Error en la transcripción: {{error}}", + "dictateNoMic": "Micrófono no disponible: {{error}}", + "pdfScannedAsImages": "{{name}} no tiene capa de texto (escaneo). Se adjuntaron {{count}} de {{total}} páginas como imágenes para un modelo de visión." }, "quickLinks": { "manageByChat": "Administrar por chat", diff --git a/core/http/react-ui/public/locales/id/chat.json b/core/http/react-ui/public/locales/id/chat.json index f647946e7..1cd45b9d6 100644 --- a/core/http/react-ui/public/locales/id/chat.json +++ b/core/http/react-ui/public/locales/id/chat.json @@ -103,7 +103,8 @@ "toasts": { "selectModel": "Silahkan pilih model", "copied": "Berhasil disalin ke papan klip", - "copyFailed": "Gagal menyalin ke papan klip" + "copyFailed": "Gagal menyalin ke papan klip", + "pdfScannedAsImages": "{{name}} tidak memiliki lapisan teks (pindaian). {{count}} dari {{total}} halaman dilampirkan sebagai gambar untuk model visi." }, "menu": { "trigger": "Obrolan", diff --git a/core/http/react-ui/public/locales/id/home.json b/core/http/react-ui/public/locales/id/home.json index ce24c2391..63e3cc6b7 100644 --- a/core/http/react-ui/public/locales/id/home.json +++ b/core/http/react-ui/public/locales/id/home.json @@ -35,7 +35,14 @@ "enterToSend": "Enter untuk mengirim", "selectModelFirst": "Pilih model terlebih dahulu", "sendMessage": "Kirim pesan", - "selectModelToast": "Silahkan pilih model terlebih dahulu" + "selectModelToast": "Silahkan pilih model terlebih dahulu", + "dictate": "Dikte (suara ke teks)", + "dictateStop": "Hentikan rekaman", + "dictateBusy": "Mentranskripsi…", + "dictateEmpty": "Tidak ada ucapan yang dikenali.", + "dictateFailed": "Transkripsi gagal: {{error}}", + "dictateNoMic": "Mikrofon tidak tersedia: {{error}}", + "pdfScannedAsImages": "{{name}} tidak memiliki lapisan teks (pindaian). {{count}} dari {{total}} halaman dilampirkan sebagai gambar untuk model visi." }, "quickLinks": { "manageByChat": "Kelola melalui obrolan", diff --git a/core/http/react-ui/public/locales/it/chat.json b/core/http/react-ui/public/locales/it/chat.json index 6c65b1c1d..3d3221d14 100644 --- a/core/http/react-ui/public/locales/it/chat.json +++ b/core/http/react-ui/public/locales/it/chat.json @@ -103,7 +103,8 @@ "toasts": { "selectModel": "Seleziona un modello", "copied": "Copiato negli appunti", - "copyFailed": "Impossibile copiare negli appunti" + "copyFailed": "Impossibile copiare negli appunti", + "pdfScannedAsImages": "{{name}} non ha un livello di testo (scansione). Allegate {{count}} di {{total}} pagine come immagini per un modello di visione." }, "menu": { "trigger": "Chat", diff --git a/core/http/react-ui/public/locales/it/home.json b/core/http/react-ui/public/locales/it/home.json index 65d2dbba7..5c21132f3 100644 --- a/core/http/react-ui/public/locales/it/home.json +++ b/core/http/react-ui/public/locales/it/home.json @@ -35,7 +35,14 @@ "enterToSend": "Invio per inviare", "selectModelFirst": "Seleziona prima un modello", "sendMessage": "Invia messaggio", - "selectModelToast": "Seleziona prima un modello" + "selectModelToast": "Seleziona prima un modello", + "dictate": "Detta (voce in testo)", + "dictateStop": "Interrompi registrazione", + "dictateBusy": "Trascrizione…", + "dictateEmpty": "Nessun parlato riconosciuto.", + "dictateFailed": "Trascrizione non riuscita: {{error}}", + "dictateNoMic": "Microfono non disponibile: {{error}}", + "pdfScannedAsImages": "{{name}} non ha un livello di testo (scansione). Allegate {{count}} di {{total}} pagine come immagini per un modello di visione." }, "quickLinks": { "manageByChat": "Gestisci via chat", diff --git a/core/http/react-ui/public/locales/ko/chat.json b/core/http/react-ui/public/locales/ko/chat.json index 8221d764e..5d7063315 100644 --- a/core/http/react-ui/public/locales/ko/chat.json +++ b/core/http/react-ui/public/locales/ko/chat.json @@ -103,7 +103,8 @@ "toasts": { "selectModel": "모델을 선택해 주세요", "copied": "클립보드에 복사되었습니다", - "copyFailed": "클립보드에 복사할 수 없습니다" + "copyFailed": "클립보드에 복사할 수 없습니다", + "pdfScannedAsImages": "{{name}}에 텍스트 레이어가 없습니다(스캔). 비전 모델용으로 {{total}}페이지 중 {{count}}페이지를 이미지로 첨부했습니다." }, "menu": { "trigger": "채팅", diff --git a/core/http/react-ui/public/locales/ko/home.json b/core/http/react-ui/public/locales/ko/home.json index beba986ed..e240be633 100644 --- a/core/http/react-ui/public/locales/ko/home.json +++ b/core/http/react-ui/public/locales/ko/home.json @@ -35,7 +35,14 @@ "enterToSend": "Enter로 전송", "selectModelFirst": "먼저 모델을 선택하세요", "sendMessage": "메시지 보내기", - "selectModelToast": "먼저 모델을 선택해 주세요" + "selectModelToast": "먼저 모델을 선택해 주세요", + "dictate": "받아쓰기 (음성을 텍스트로)", + "dictateStop": "녹음 중지", + "dictateBusy": "변환 중…", + "dictateEmpty": "인식된 음성이 없습니다.", + "dictateFailed": "변환 실패: {{error}}", + "dictateNoMic": "마이크를 사용할 수 없습니다: {{error}}", + "pdfScannedAsImages": "{{name}}에 텍스트 레이어가 없습니다(스캔). 비전 모델용으로 {{total}}페이지 중 {{count}}페이지를 이미지로 첨부했습니다." }, "quickLinks": { "manageByChat": "채팅으로 관리", diff --git a/core/http/react-ui/public/locales/pt-BR/chat.json b/core/http/react-ui/public/locales/pt-BR/chat.json index 7f0aad636..4be78db8e 100644 --- a/core/http/react-ui/public/locales/pt-BR/chat.json +++ b/core/http/react-ui/public/locales/pt-BR/chat.json @@ -107,7 +107,8 @@ "copied": "Copiado para a área de transferência", "copyFailed": "Não foi possível copiar para a área de transferência", "chatCopied": "Chat copiado para a área de transferência", - "forked": "Novo chat criado" + "forked": "Novo chat criado", + "pdfScannedAsImages": "{{name}} não tem camada de texto (digitalização). {{count}} de {{total}} páginas anexadas como imagens para um modelo de visão." }, "menu": { "trigger": "Chats", diff --git a/core/http/react-ui/public/locales/pt-BR/home.json b/core/http/react-ui/public/locales/pt-BR/home.json index 3a8e39d38..cbcdb42c7 100644 --- a/core/http/react-ui/public/locales/pt-BR/home.json +++ b/core/http/react-ui/public/locales/pt-BR/home.json @@ -35,7 +35,14 @@ "enterToSend": "Enter para enviar", "selectModelFirst": "Selecione um modelo primeiro", "sendMessage": "Enviar mensagem", - "selectModelToast": "Selecione um modelo primeiro" + "selectModelToast": "Selecione um modelo primeiro", + "dictate": "Ditar (voz para texto)", + "dictateStop": "Parar gravação", + "dictateBusy": "Transcrevendo…", + "dictateEmpty": "Nenhuma fala reconhecida.", + "dictateFailed": "Falha na transcrição: {{error}}", + "dictateNoMic": "Microfone indisponível: {{error}}", + "pdfScannedAsImages": "{{name}} não tem camada de texto (digitalização). {{count}} de {{total}} páginas anexadas como imagens para um modelo de visão." }, "quickLinks": { "manageByChat": "Gerenciar por chat", diff --git a/core/http/react-ui/public/locales/zh-CN/chat.json b/core/http/react-ui/public/locales/zh-CN/chat.json index 3708f952e..71a6b5590 100644 --- a/core/http/react-ui/public/locales/zh-CN/chat.json +++ b/core/http/react-ui/public/locales/zh-CN/chat.json @@ -103,7 +103,8 @@ "toasts": { "selectModel": "请选择一个模型", "copied": "已复制到剪贴板", - "copyFailed": "无法复制到剪贴板" + "copyFailed": "无法复制到剪贴板", + "pdfScannedAsImages": "{{name}} 没有文本层（扫描件）。已将 {{total}} 页中的 {{count}} 页作为图片附加，供视觉模型使用。" }, "menu": { "trigger": "聊天", diff --git a/core/http/react-ui/public/locales/zh-CN/home.json b/core/http/react-ui/public/locales/zh-CN/home.json index 9ab886ee5..e6af6c163 100644 --- a/core/http/react-ui/public/locales/zh-CN/home.json +++ b/core/http/react-ui/public/locales/zh-CN/home.json @@ -35,7 +35,14 @@ "enterToSend": "按回车发送", "selectModelFirst": "请先选择一个模型", "sendMessage": "发送消息", - "selectModelToast": "请先选择一个模型" + "selectModelToast": "请先选择一个模型", + "dictate": "听写（语音转文字）", + "dictateStop": "停止录音", + "dictateBusy": "正在转写…", + "dictateEmpty": "未识别到语音。", + "dictateFailed": "转写失败：{{error}}", + "dictateNoMic": "麦克风不可用：{{error}}", + "pdfScannedAsImages": "{{name}} 没有文本层（扫描件）。已将 {{total}} 页中的 {{count}} 页作为图片附加，供视觉模型使用。" }, "quickLinks": { "manageByChat": "通过聊天管理", diff --git a/core/http/react-ui/src/components/home/DictateButton.jsx b/core/http/react-ui/src/components/home/DictateButton.jsx new file mode 100644 index 000000000..d9448396e --- /dev/null +++ b/core/http/react-ui/src/components/home/DictateButton.jsx @@ -0,0 +1,94 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import Icon from '../Icon' +import { useModels } from '../../hooks/useModels' +import { CAP_TRANSCRIPT } from '../../utils/capabilities' +import { audioApi } from '../../utils/api' + +// Dictation for the composer: the first click records from the microphone, +// the second stops and sends the recording to /v1/audio/transcriptions with +// the first installed transcription model. The text is handed to onText; the +// user reads and sends it like any typed message. Without a transcription +// model, or without a microphone API in the browser, the button is not shown. +export default function DictateButton({ onText, onError }) { + const { t } = useTranslation('home') + const { models } = useModels(CAP_TRANSCRIPT) + const [state, setState] = useState('idle') // idle | recording | transcribing + const recorder = useRef(null) + const chunks = useRef([]) + const model = models[0]?.id + + // Release the microphone when the composer goes away mid-recording. + useEffect(() => () => { + const r = recorder.current + if (r && r.state !== 'inactive') { + r.onstop = null + r.stop() + } + r?.stream?.getTracks().forEach(tr => tr.stop()) + }, []) + + const transcribe = useCallback(async (blob) => { + setState('transcribing') + try { + const fd = new FormData() + fd.append('file', blob, `dictation.${(blob.type.split('/')[1] || 'webm').split(';')[0]}`) + fd.append('model', model) + const data = await audioApi.transcribe(fd) + const text = (data?.text || '').trim() + if (text) onText(text) + else onError?.(t('input.dictateEmpty')) + } catch (err) { + onError?.(t('input.dictateFailed', { error: err?.message || String(err) })) + } finally { + setState('idle') + } + }, [model, onText, onError, t]) + + const toggle = useCallback(async () => { + if (state === 'recording') { + recorder.current?.stop() + return + } + if (state !== 'idle') return + let stream + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch (err) { + onError?.(t('input.dictateNoMic', { error: err?.message || String(err) })) + return + } + const r = new MediaRecorder(stream) + chunks.current = [] + r.ondataavailable = (e) => { if (e.data?.size) chunks.current.push(e.data) } + r.onstop = () => { + stream.getTracks().forEach(tr => tr.stop()) + const blob = new Blob(chunks.current, { type: r.mimeType || 'audio/webm' }) + if (blob.size) transcribe(blob) + else setState('idle') + } + recorder.current = r + r.start() + setState('recording') + }, [state, transcribe, onError, t]) + + if (!model || typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === 'undefined') { + return null + } + const label = state === 'recording' ? t('input.dictateStop') : state === 'transcribing' ? t('input.dictateBusy') : t('input.dictate') + return ( + + ) +} diff --git a/core/http/react-ui/src/components/home/HomeComposer.jsx b/core/http/react-ui/src/components/home/HomeComposer.jsx index 1053c5921..23a4a300c 100644 --- a/core/http/react-ui/src/components/home/HomeComposer.jsx +++ b/core/http/react-ui/src/components/home/HomeComposer.jsx @@ -3,6 +3,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation, Trans } from 'react-i18next' import { availableActions, filterActions, SLASH_GROUPS } from './homeActions' import Icon from '../Icon' +import DictateButton from './DictateButton' // The Enter glyph is not in the kit sprite. A return arrow in a darker cell // teaches the shortcut next to the label. @@ -31,6 +32,7 @@ function Marked({ text, upto }) { // slash { actions, groups, label, desc, groupLabel } for its own menu // streaming swaps Send for a Stop button that calls onStop // onPaste, onKeyDownExtra (return true to take the key), fileAccept, rows, +// onDictateError (message for a failed dictation, e.g. a toast), // testIds and class names for the pieces tests and the thread reach for export default function HomeComposer({ message, onMessage, onSubmit, canSend, sending, sendTitle, textareaRef, @@ -38,7 +40,7 @@ export default function HomeComposer({ placeholder, slashContext, onRunAction, slash, streaming = false, onStop, stopTitle, onPaste, onKeyDownExtra, fileAccept = '.txt,.md,.pdf', rows = 3, strictEnter = false, testId = 'home-composer', - textareaTestId, textareaClass = '', sendId, sendTestId, hint, + textareaTestId, textareaClass = '', sendId, sendTestId, hint, onDictateError, }) { const { t } = useTranslation('home') const imageRef = useRef(null) @@ -204,11 +206,15 @@ export default function HomeComposer({ + onMessage(message.trim() ? `${message.trimEnd()} ${text}` : text)} + onError={onDictateError} + /> {hint || t('input.enterHint')} {streaming ? ( diff --git a/core/http/react-ui/src/pages/Chat.jsx b/core/http/react-ui/src/pages/Chat.jsx index ab420bc90..c135da6e4 100644 --- a/core/http/react-ui/src/pages/Chat.jsx +++ b/core/http/react-ui/src/pages/Chat.jsx @@ -9,7 +9,7 @@ import { extractCodeArtifacts, renderMarkdownWithArtifacts } from '../utils/arti import CanvasPanel from '../components/CanvasPanel' // eslint-disable-next-line no-unused-vars import { fileToBase64, modelsApi, mcpApi } from '../utils/api' -import { readAttachmentText } from '../utils/pdf' +import { readAttachmentText, renderPdfPages, PdfNoTextError } from '../utils/pdf' import { CAP_CHAT } from '../utils/capabilities' import { useMCPClient } from '../hooks/useMCPClient' // eslint-disable-next-line no-unused-vars @@ -715,7 +715,18 @@ export default function Chat() { if (!file.type.startsWith('image/') && !file.type.startsWith('audio/') && !file.type.startsWith('video/')) { try { entry.textContent = await readAttachmentText(file) - } catch { + } catch (err) { + // A scan has no text layer: send its pages as images for a vision model. + if (err instanceof PdfNoTextError) { + try { + const { pages, total } = await renderPdfPages(file) + newFiles.push(...pages) + addToast(t('toasts.pdfScannedAsImages', { name: file.name, count: pages.length, total }), 'info') + continue + } catch { + // fall through to the read error below + } + } addToast(t('toasts.pdfReadFailed', { name: file.name }), 'error') continue } @@ -1217,6 +1228,7 @@ export default function Chat() {
addToast(m, 'error')} message={input} onMessage={setInput} onSubmit={handleSend} diff --git a/core/http/react-ui/src/pages/Home.jsx b/core/http/react-ui/src/pages/Home.jsx index 22bbcc984..f9554c9a2 100644 --- a/core/http/react-ui/src/pages/Home.jsx +++ b/core/http/react-ui/src/pages/Home.jsx @@ -27,7 +27,7 @@ import { useResources } from '../hooks/useResources' import { usePolling } from '../hooks/usePolling' import { useOperations } from '../hooks/useOperations' import { fileToBase64, backendControlApi, systemApi, modelsApi, mcpApi, nodesApi } from '../utils/api' -import { readAttachmentText } from '../utils/pdf' +import { readAttachmentText, renderPdfPages, PdfNoTextError } from '../utils/pdf' import { greetingKey } from '../utils/greeting' import { listConversations, setActiveConversation, removeConversation, @@ -221,7 +221,18 @@ export default function Home() { if (!file.type.startsWith('image/') && !file.type.startsWith('audio/')) { try { entry.textContent = await readAttachmentText(file) - } catch { + } catch (err) { + // A scan has no text layer: send its pages as images for a vision model. + if (err instanceof PdfNoTextError) { + try { + const { pages, total } = await renderPdfPages(file) + setImageFiles(prev => [...prev, ...pages]) + addToast(t('input.pdfScannedAsImages', { name: file.name, count: pages.length, total }), 'info') + continue + } catch { + // fall through to the read error below + } + } addToast(t('input.pdfReadFailed', { name: file.name }), 'error') continue } @@ -551,6 +562,7 @@ export default function Home() {
addToast(m, 'error')} message={message} onMessage={setMessage} onSubmit={doSubmit} diff --git a/core/http/react-ui/src/utils/pdf.js b/core/http/react-ui/src/utils/pdf.js index 2996e50db..ee47e5ddf 100644 --- a/core/http/react-ui/src/utils/pdf.js +++ b/core/http/react-ui/src/utils/pdf.js @@ -13,6 +13,15 @@ async function loadPdfjs() { return pdfjs } +// Thrown when a PDF parses but carries no text layer (a scan). Callers can +// fall back to renderPdfPages and send the pages as images instead. +export class PdfNoTextError extends Error { + constructor() { + super('PDF has no extractable text') + this.name = 'PdfNoTextError' + } +} + // Returns the text layer of every page, one block per page. Throws when the // file cannot be parsed or has no text layer (scanned PDFs): sending an empty // attachment to the model would look like success and silently lose the file. @@ -33,7 +42,7 @@ export async function extractPdfText(file) { pages.push(text.trim()) } const text = pages.filter(Boolean).join('\n\n') - if (!text) throw new Error('PDF has no extractable text') + if (!text) throw new PdfNoTextError() return text } finally { await doc.destroy() @@ -46,3 +55,32 @@ export async function readAttachmentText(file) { if (isPdf(file)) return extractPdfText(file) return file.text().catch(() => '') } + +// Pages of a scanned PDF as PNG attachments ({ name, type, base64 }), so a +// vision model can read them. Renders at most maxPages pages; `total` tells the +// caller how many the document has. Scale 1.5 keeps small print legible at a +// size the vision encoders take. +export const SCANNED_PDF_MAX_PAGES = 10 +export async function renderPdfPages(file, { maxPages = SCANNED_PDF_MAX_PAGES, scale = 1.5 } = {}) { + const pdfjs = await loadPdfjs() + const data = new Uint8Array(await file.arrayBuffer()) + const doc = await pdfjs.getDocument({ data }).promise + try { + const stem = (file.name || 'document.pdf').replace(/\.pdf$/i, '') + const pages = [] + const n = Math.min(doc.numPages, maxPages) + for (let i = 1; i <= n; i++) { + const page = await doc.getPage(i) + const viewport = page.getViewport({ scale }) + const canvas = document.createElement('canvas') + canvas.width = Math.ceil(viewport.width) + canvas.height = Math.ceil(viewport.height) + await page.render({ canvasContext: canvas.getContext('2d'), viewport }).promise + const url = canvas.toDataURL('image/png') + pages.push({ name: `${stem}-page-${i}.png`, type: 'image/png', base64: url.split(',')[1] }) + } + return { pages, total: doc.numPages } + } finally { + await doc.destroy() + } +} diff --git a/docs/content/features/chat.md b/docs/content/features/chat.md index 4b2c96515..cb34c775b 100644 --- a/docs/content/features/chat.md +++ b/docs/content/features/chat.md @@ -46,6 +46,10 @@ The **MCP** chip opens the server and client tool lists. **Canvas** turns code b Enter sends, Shift+Enter adds a line. Paste an image to attach it. Under the box, the line shows the speed while a reply streams and the token count of the chat. +The buttons under the box attach an image, an audio file or a document. A PDF is sent as its text layer. A scanned PDF has none, so its first ten pages are attached as images instead, for a model that reads images; a toast says how many pages went along. + +The microphone button dictates: click it to record, click again to stop, and the recording goes to `/v1/audio/transcriptions` with the first installed transcription model ([audio to text]({{< relref "audio-to-text.md" >}})). The text lands in the box, so you can read and correct it before sending. Without a transcription model the button is not shown. Recording needs microphone access in the browser, which browsers only grant on `https://` or `localhost`. + ## Conversations Press `Ctrl+K` (`Cmd+K`) or **Chats** to open the list of conversations, grouped by day like **Jump back in** on Home. Type to search names and message text. Arrow keys move, `Enter` opens, `F2` renames and `Delete` removes. Each row also offers rename, duplicate, copy and export. Removing a chat hides it and shows an **Undo** toast; the chat is deleted for good when the toast goes away. The name in the header can be renamed with a click.