From e684fdf084c7cedd1c3ede4e8584e2ecf39fcc34 Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Tue, 20 Oct 2026 18:08:37 +0200 Subject: [PATCH] fix(ui): reuse Home picker for Sound expansion (#12790) * fix(ui): reuse Home picker for Sound expansion Use the shared model chip and listbox for Sound expansion instead of a native select. Keep explicit Sound selection and Backend default independent from Home model preferences. Preserve keyboard navigation and expose the selected value through an accessible description. Hide runtime badges without loaded-model data. Assisted-by: nib:gpt-6.1-sol Signed-off-by: Ettore Di Giacinto * test(ui): select Sound expansion through shared picker Update media history coverage to use the model button and listbox. Assisted-by: nib:gpt-6.1-sol Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/http/react-ui/e2e/home-console.spec.js | 13 +- core/http/react-ui/e2e/media-history.spec.js | 3 +- .../http/react-ui/e2e/sound-expansion.spec.js | 156 ++++++++++++++++-- .../src/components/home/HomeModelPicker.jsx | 77 +++++---- core/http/react-ui/src/pages/Sound.jsx | 24 ++- docs/content/features/text-to-audio.md | 5 +- 6 files changed, 223 insertions(+), 55 deletions(-) diff --git a/core/http/react-ui/e2e/home-console.spec.js b/core/http/react-ui/e2e/home-console.spec.js index 42a5c677d..af48f675c 100644 --- a/core/http/react-ui/e2e/home-console.spec.js +++ b/core/http/react-ui/e2e/home-console.spec.js @@ -174,6 +174,17 @@ test.describe('Home model chip', () => { await expect(chip).toBeFocused() }) + test('keeps Home auto-selection and remembers an explicit model', async ({ page }) => { + await ready(page) + const chip = page.getByTestId('home-model-chip') + await expect(chip).toContainText('qwen3-8b-instruct') + await chip.click() + await page.getByRole('option', { name: /gemma-4-e4b/ }).click() + expect(await page.evaluate(() => localStorage.getItem('localai_last_model:FLAG_CHAT'))).toBe('gemma-4-e4b-it-qat-q4_0') + await page.reload() + await expect(chip).toContainText('gemma-4-e4b-it-qat-q4_0') + }) + test('Escape closes the list and returns to the chip', async ({ page }) => { await ready(page) const chip = page.getByTestId('home-model-chip') @@ -479,7 +490,7 @@ test.describe('Home first run', () => { await mockFirstRun(page) await ready(page) const guide = page.getByTestId('home-first-run') - await expect(guide.getByRole('listitem')).toHaveCount(3) + await expect(guide.locator('.home-step')).toHaveCount(3) await expect(guide.locator('.home-step[data-state="now"]')).toContainText('Browse the Model Gallery') await expect(guide.locator('.home-step').nth(1).locator('.home-starters')).toContainText('tiny-chat') await expect(guide.getByRole('button', { name: /browse model gallery/i })).toBeVisible() diff --git a/core/http/react-ui/e2e/media-history.spec.js b/core/http/react-ui/e2e/media-history.spec.js index 216506d89..1b9c84fd0 100644 --- a/core/http/react-ui/e2e/media-history.spec.js +++ b/core/http/react-ui/e2e/media-history.spec.js @@ -230,7 +230,8 @@ test.describe('Media History - Sound Generation', () => { await page.goto('/app/sound') await expect(page.getByRole('button', { name: 'test-sound-model' })).toBeVisible({ timeout: 10_000 }) - await page.getByLabel('Prompt expansion model', { exact: true }).selectOption('__backend_default__') + await page.getByRole('button', { name: 'Prompt expansion model', exact: true }).click() + await page.getByRole('option', { name: 'Backend default', exact: true }).click() await page.locator('.textarea').first().fill('upbeat jazz') await page.locator('button[type="submit"]').click() diff --git a/core/http/react-ui/e2e/sound-expansion.spec.js b/core/http/react-ui/e2e/sound-expansion.spec.js index b1dbef445..cfbba10f7 100644 --- a/core/http/react-ui/e2e/sound-expansion.spec.js +++ b/core/http/react-ui/e2e/sound-expansion.spec.js @@ -6,7 +6,12 @@ const preference = 'localai_sound_expansion_model' const chatPreference = 'localai_last_model:FLAG_CHAT' const audio = { id: 'music', capabilities: ['FLAG_SOUND_GENERATION'] } const chat = id => ({ id, capabilities: ['FLAG_CHAT'] }) -const picker = page => page.getByLabel('Prompt expansion model', { exact: true }) +const picker = page => page.getByRole('button', { name: 'Prompt expansion model', exact: true }) +const choiceLabel = value => value === '__backend_default__' ? 'Backend default' : value || 'Choose a model or backend default' +async function choose(page, value) { + await picker(page).click() + await page.getByRole('listbox', { name: 'Prompt expansion model' }).getByRole('option').filter({ has: page.locator('code').getByText(choiceLabel(value), { exact: true }) }).click() +} const submit = page => page.locator('[data-testid="ws-compose"] button[type="submit"]') async function setup(page, models = [audio, chat('writer'), chat('poet')], saved) { await page.route('**/api/**', route => route.fulfill({ json: {} })) @@ -33,13 +38,15 @@ test('requires a choice, filters installed chat models and remembers only the So const requests = await setup(page, [audio, chat('writer'), chat('poet'), { id: 'embedding', capabilities: ['FLAG_EMBEDDINGS'] }, { ...chat('disabled'), disabled: true }]) await prompt(page) await expect(submit(page)).toBeDisabled() - await expect(picker(page).locator('option')).toHaveText(['Choose a model or backend default', 'Backend default', 'writer', 'poet']) - await picker(page).selectOption('poet') + await picker(page).click() + await expect(page.getByRole('listbox', { name: 'Prompt expansion model' }).getByRole('option').locator('code')).toHaveText(['Backend default', 'writer', 'poet']) + await page.keyboard.press('Escape') + await choose(page, 'poet') await submit(page).click() await expect.poll(() => requests.length).toBe(1) expect(requests[0]).toMatchObject({ model_id: 'music', text: 'A quiet piano instrumental', instrumental: false, expansion_model: 'poet' }) await page.reload() - await expect(picker(page)).toHaveValue('poet') + await expect(picker(page)).toContainText(choiceLabel('poet')) expect(await page.evaluate(key => localStorage.getItem(key), chatPreference)).toBe('general-chat') expect(await page.evaluate(key => localStorage.getItem(key), preference)).toBe('poet') }) @@ -47,7 +54,7 @@ for (const stale of ['removed', 'embedding']) { test(`clears unavailable remembered model ${stale} and asks for a choice`, async ({ page }) => { await setup(page, [audio, chat('writer'), { id: 'embedding', capabilities: ['FLAG_EMBEDDINGS'] }], stale) await prompt(page) - await expect(picker(page)).toHaveValue('') + await expect(picker(page)).toContainText(choiceLabel('')) await expect(submit(page)).toBeDisabled() await expect(page.getByText('Choose a chat model or Backend default to continue.', { exact: true })).toBeVisible() expect(await page.evaluate(key => localStorage.getItem(key), preference)).toBeNull() @@ -55,10 +62,10 @@ for (const stale of ['removed', 'embedding']) { } test('revalidates a selected model when the installed list changes', async ({ page }) => { await setup(page, [audio, chat('writer'), chat('poet')], 'writer') - await expect(picker(page)).toHaveValue('writer') + await expect(picker(page)).toContainText(choiceLabel('writer')) await page.route('**/api/models/capabilities', route => route.fulfill({ json: { data: [audio, chat('poet')] } })) await page.evaluate(() => window.dispatchEvent(new Event('focus'))) - await expect(picker(page)).toHaveValue('') + await expect(picker(page)).toContainText(choiceLabel('')) await prompt(page) await expect(submit(page)).toBeDisabled() expect(await page.evaluate(key => localStorage.getItem(key), preference)).toBeNull() @@ -68,17 +75,17 @@ test('Backend default works without chat models, omits expansion_model and survi await prompt(page) await expect(submit(page)).toBeDisabled() await expect(page.getByText('No installed chat models. Choose Backend default to use the audio backend.', { exact: true })).toBeVisible() - await picker(page).selectOption('__backend_default__') + await choose(page, '__backend_default__') await submit(page).click() await expect.poll(() => requests.length).toBe(1) expect(requests[0]).not.toHaveProperty('expansion_model') await page.reload() - await expect(picker(page)).toHaveValue('__backend_default__') + await expect(picker(page)).toContainText(choiceLabel('__backend_default__')) expect(await page.evaluate(key => localStorage.getItem(key), chatPreference)).toBe('general-chat') }) test('Advanced preserves all fields and never sends the remembered expansion model', async ({ page }) => { const requests = await setup(page, undefined, 'writer') - await expect(picker(page)).toHaveValue('writer') + await expect(picker(page)).toContainText(choiceLabel('writer')) await page.getByRole('tab', { name: 'Advanced', exact: true }).click() await expect(picker(page)).toHaveCount(0) await page.locator('#sound-caption').fill('Warm acoustic folk') @@ -117,15 +124,15 @@ test('Re-run with edits restores the original expansion choice and tracks edits' contentType: 'audio/wav', headers: { 'Content-Disposition': 'attachment; filename="piano.wav"' }, body: silentWav(), })) await prompt(page) - await picker(page).selectOption('writer') + await choose(page, 'writer') await submit(page).click() const tile = page.getByTestId('media-history-item').first() await expect(tile).toBeVisible() - await picker(page).selectOption('poet') + await choose(page, 'poet') await tile.locator('.ws-tile__open').click() await page.getByTestId('ws-rerun').click() - await expect(picker(page)).toHaveValue('writer') - await picker(page).selectOption('poet') + await expect(picker(page)).toContainText(choiceLabel('writer')) + await choose(page, 'poet') await expect(page.getByTestId('ws-changes')).toContainText('Prompt expansion model writer to poet') }) @@ -139,8 +146,127 @@ for (const historical of [undefined, 'removed']) { await page.reload() await page.getByTestId('media-history-item').first().locator('.ws-tile__open').click() await page.getByTestId('ws-rerun').click() - await expect(picker(page)).toHaveValue(historical ? '' : '__backend_default__') + await expect(picker(page)).toContainText(choiceLabel(historical ? '' : '__backend_default__')) if (historical) await expect(submit(page)).toBeDisabled() else await expect(submit(page)).toBeEnabled() }) } + +test('shared chip supports keyboard, focus return and dismissal without choosing', async ({ page }) => { + await setup(page) + const chip = picker(page) + await expect(chip).toHaveAttribute('id', 'sound-expansion-model') + await expect(chip).toHaveAttribute('aria-describedby', /(?:^| )sound-expansion-help(?: |$)/) + await chip.press('ArrowDown') + const list = page.getByRole('listbox', { name: 'Prompt expansion model' }) + await expect(list).toBeVisible() + await page.keyboard.press('ArrowDown') + await page.keyboard.press('Enter') + await expect(chip).toContainText('writer') + await expect(chip).toBeFocused() + await chip.click() + await page.keyboard.press('Escape') + await expect(list).toHaveCount(0) + await expect(chip).toBeFocused() + await chip.click() + await page.keyboard.press('Tab') + await expect(list).toHaveCount(0) + await chip.click() + await page.locator('[data-testid="ws-compose"] textarea').first().click() + await expect(list).toHaveCount(0) + await expect(chip).toContainText('writer') +}) + +for (const theme of ['dark', 'light']) { + test(`searches long IDs without clipping on a narrow ${theme} viewport`, async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }) + await setup(page, [audio, ...Array.from({ length: 8 }, (_, i) => chat(`writer-${i}-${'long-model-name-'.repeat(5)}`))]) + await page.evaluate(theme => document.documentElement.setAttribute('data-theme', theme), theme) + await picker(page).click() + const search = page.getByRole('combobox') + await search.fill('no-such-model') + await expect(page.getByRole('option')).toHaveCount(0) + await expect(page.getByText('No model matches.', { exact: false })).toBeVisible() + await search.fill('writer-7') + await expect(page.getByRole('option')).toHaveCount(1) + const menu = page.locator('.home-menu--models') + const box = await menu.boundingBox() + expect(box.x).toBeGreaterThanOrEqual(0) + expect(box.x + box.width).toBeLessThanOrEqual(390) + await page.keyboard.press('Enter') + await expect(picker(page)).toContainText('writer-7') + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390) + if (process.env.SOUND_PICKER_EVIDENCE) { + await picker(page).click() + await page.screenshot({ path: `${process.env.SOUND_PICKER_EVIDENCE}/sound-${theme}-narrow.png`, fullPage: true }) + } + }) +} + +test('Backend default remains usable after a model list error', async ({ page }) => { + const requests = await setup(page) + await page.route('**/api/models/capabilities', route => route.fulfill({ status: 500, json: { error: 'offline' } })) + await page.route('**/v1/models', route => route.fulfill({ status: 500, json: { error: 'offline' } })) + await page.evaluate(() => window.dispatchEvent(new Event('focus'))) + await expect(page.locator('#sound-expansion-help')).toHaveText('Cannot load chat models. Choose Backend default or return to this tab to retry.') + await choose(page, '__backend_default__') + await prompt(page) + await expect(submit(page)).toBeEnabled() + await submit(page).click() + await expect.poll(() => requests.length).toBe(1) + expect(requests[0]).not.toHaveProperty('expansion_model') +}) + + +test('initial model loading disables the picker until the response resolves', async ({ page }) => { + const requests = await setup(page, undefined, 'writer') + let release + const responseReady = new Promise(resolve => { release = resolve }) + await page.route('**/api/models/capabilities', async route => { + await responseReady + await route.fulfill({ json: { data: [audio, chat('writer'), chat('poet')] } }) + }) + await page.reload() + try { + await expect(picker(page)).toBeDisabled() + await expect(page.locator('#sound-expansion-help')).toHaveText('Loading installed chat models…') + await prompt(page) + await expect(submit(page)).toBeDisabled() + } finally { + release() + } + await expect(picker(page)).toBeEnabled() + await expect(picker(page)).toContainText('writer') + await choose(page, 'poet') + await expect(submit(page)).toBeEnabled() + await submit(page).click() + await expect.poll(() => requests.length).toBe(1) + expect(requests[0]).toHaveProperty('expansion_model', 'poet') +}) + +test('Sound shows no unverified load status on any expansion model row or the chip', async ({ page }) => { + await setup(page) + await picker(page).click() + const options = page.getByRole('listbox', { name: 'Prompt expansion model' }).getByRole('option') + await expect(options.locator('code')).toHaveText(['Backend default', 'writer', 'poet']) + await expect(options.locator('.home-dot')).toHaveCount(0) + await expect(options.locator('small')).toHaveCount(0) + await page.keyboard.press('Escape') + await choose(page, 'writer') + await expect(picker(page).locator('.home-dot')).toHaveCount(0) +}) + +test('the stable picker label describes its current choice and retains the help association', async ({ page }) => { + await setup(page) + const chip = picker(page) + const help = 'A chat model expands your description into style and lyrics. Backend default uses the audio backend’s existing prompt behavior.' + await expect(chip).toHaveAccessibleName('Prompt expansion model') + await expect(chip).toHaveAccessibleDescription(`${choiceLabel('')} ${help}`) + await choose(page, 'writer') + await expect(chip).toHaveAccessibleName('Prompt expansion model') + await expect(chip).toHaveAccessibleDescription(`writer ${help}`) + await choose(page, '__backend_default__') + await expect(chip).toHaveAccessibleName('Prompt expansion model') + await expect(chip).toHaveAccessibleDescription(`Backend default ${help}`) + await expect(chip).toHaveAttribute('aria-describedby', /(?:^| )sound-expansion-help(?: |$)/) +}) diff --git a/core/http/react-ui/src/components/home/HomeModelPicker.jsx b/core/http/react-ui/src/components/home/HomeModelPicker.jsx index d65ca82e3..ccd92ca3f 100644 --- a/core/http/react-ui/src/components/home/HomeModelPicker.jsx +++ b/core/http/react-ui/src/components/home/HomeModelPicker.jsx @@ -1,5 +1,5 @@ // eslint-disable-next-line no-unused-vars -import { Fragment, forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react' +import { Fragment, forwardRef, useCallback, useEffect, useImperativeHandle, useId, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useModels } from '../../hooks/useModels' import { readLastModel, writeLastModel } from '../../utils/lastModel' @@ -25,7 +25,9 @@ const HomeModelPicker = forwardRef(function HomeModelPicker( { value, onChange, capability, loadedIds, disabled = false, placeholder, models: givenModels, loading: givenLoading, grouped = false, describe, onOpen, footer, - showWarm = true, labels, + showWarm = true, labels, autoSelect = true, rememberSelection = true, + defaultChoice, defaultSelected = false, onChooseDefault, id, describedBy, + }, ref, ) { @@ -34,6 +36,10 @@ const HomeModelPicker = forwardRef(function HomeModelPicker( const models = givenModels || own.models const loading = givenModels ? !!givenLoading : own.loading const names = useMemo(() => models.map(m => m.id), [models]) + const instanceId = useId() + const menuId = `${instanceId}-menu` + const listId = `${instanceId}-list` + // A null row is a non-model action, never an installed model ID. const [open, setOpen] = useState(false) const [query, setQuery] = useState('') const [active, setActive] = useState(0) @@ -46,19 +52,19 @@ const HomeModelPicker = forwardRef(function HomeModelPicker( // there, otherwise the first one. Auto-select is not a user choice, so it is // not written back. useEffect(() => { - if (names.length > 0 && (!value || !names.includes(value))) { + if (autoSelect && !defaultSelected && names.length > 0 && (!value || !names.includes(value))) { const remembered = readLastModel(capability) onChange(remembered && names.includes(remembered) ? remembered : names[0]) } - }, [names, value, onChange, capability]) + }, [names, value, onChange, capability, autoSelect, defaultSelected]) // Loaded models first when the list is grouped; each group keeps its order. const shown = useMemo(() => { const q = query.trim().toLowerCase() const found = q ? names.filter(n => n.toLowerCase().includes(q)) : names - if (!grouped) return found - return [...found.filter(n => loadedIds?.has(n)), ...found.filter(n => !loadedIds?.has(n))] - }, [names, query, grouped, loadedIds]) + const ordered = grouped ? [...found.filter(n => loadedIds?.has(n)), ...found.filter(n => !loadedIds?.has(n))] : found + return defaultChoice && (!q || defaultChoice.toLowerCase().includes(q)) ? [null, ...ordered] : ordered + }, [names, query, grouped, loadedIds, defaultChoice]) const close = useCallback((restoreFocus = false) => { setOpen(false) @@ -67,12 +73,12 @@ const HomeModelPicker = forwardRef(function HomeModelPicker( }, []) const openMenu = useCallback(() => { - if (disabled || names.length === 0) return - setActive(Math.max(0, names.indexOf(value))) + if (disabled || (names.length === 0 && !defaultChoice)) return + setActive(defaultSelected ? 0 : Math.max(0, names.indexOf(value) + (defaultChoice ? 1 : 0))) setQuery('') setOpen(true) onOpen?.() - }, [disabled, names, value, onOpen]) + }, [disabled, names, value, onOpen, defaultChoice, defaultSelected]) useImperativeHandle(ref, () => ({ open: openMenu, focus: () => chipRef.current?.focus() }), [openMenu]) @@ -97,10 +103,13 @@ const HomeModelPicker = forwardRef(function HomeModelPicker( }, [open, active, shown]) const pick = useCallback((name) => { - writeLastModel(capability, name) - onChange(name) + if (name === null) onChooseDefault?.() + else { + if (rememberSelection) writeLastModel(capability, name) + onChange(name) + } close(true) - }, [capability, onChange, close]) + }, [capability, onChange, close, rememberSelection, onChooseDefault]) const onMenuKey = (e) => { const n = shown.length @@ -108,24 +117,27 @@ const HomeModelPicker = forwardRef(function HomeModelPicker( else if (e.key === 'ArrowUp') { e.preventDefault(); if (n) setActive(a => (a + n - 1) % n) } else if (e.key === 'Home' && !query) { e.preventDefault(); setActive(0) } else if (e.key === 'End' && !query) { e.preventDefault(); setActive(Math.max(0, n - 1)) } - else if (e.key === 'Enter') { e.preventDefault(); if (shown[active]) pick(shown[active]) } + else if (e.key === 'Enter') { e.preventDefault(); if (active < n) pick(shown[active]) } else if (e.key === 'Escape') { e.preventDefault(); e.stopPropagation(); close(true) } else if (e.key === 'Tab') close() } const isWarm = (name) => loadedIds?.has(name) - const label = value || placeholder || (loading ? t('picker.loading') : (labels?.none || t('picker.none'))) - const activeId = shown[active] ? `home-model-opt-${active}` : undefined + const label = (defaultSelected ? defaultChoice : value) || placeholder || (loading ? t('picker.loading') : (labels?.none || t('picker.none'))) + const activeId = active < shown.length ? `${instanceId}-opt-${active}` : undefined return (
{open && ( -
+
{names.length >= FILTER_FROM && ( { setQuery(e.target.value); setActive(0) }} role="combobox" aria-expanded="true" - aria-controls="home-model-list" + aria-controls={listId} aria-activedescendant={activeId} /> )}
    {t('picker.noMatch')}} {shown.map((name, i) => { const warm = isWarm(name) - const info = describe ? describe(name, warm) : null + const info = name !== null && describe ? describe(name, warm) : null + const selected = name === null ? defaultSelected : !defaultSelected && name === value const first = grouped && (i === 0 || isWarm(shown[i - 1]) !== warm) return ( - + {first && (
  • {warm ? t('picker.loadedNow') : t('picker.installed')}
    • )}
  • setActive(i)} onClick={() => pick(name)} > - {showWarm &&
    • diff --git a/core/http/react-ui/src/pages/Sound.jsx b/core/http/react-ui/src/pages/Sound.jsx index 0877c9335..15095fa91 100644 --- a/core/http/react-ui/src/pages/Sound.jsx +++ b/core/http/react-ui/src/pages/Sound.jsx @@ -1,5 +1,7 @@ import { useState } from 'react' // eslint-disable-next-line no-unused-vars +import HomeModelPicker from '../components/home/HomeModelPicker' +// eslint-disable-next-line no-unused-vars import RequestPanel from '../components/RequestPanel' import { useParams, useOutletContext } from 'react-router-dom' import { CAP_SOUND_GENERATION } from '../utils/capabilities' @@ -218,11 +220,23 @@ export default function Sound() { /> {!text && } - + expansion.choose(BACKEND_DEFAULT)} + />

    {expansion.loading ? t('sound.expansion.loading') : expansion.error ? t('sound.expansion.error') diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md index 17d013630..f73e1e50d 100644 --- a/docs/content/features/text-to-audio.md +++ b/docs/content/features/text-to-audio.md @@ -1170,7 +1170,10 @@ tool call, multiple calls, or invalid arguments returns a request error. Expansi Cancellation propagates to model loading, text inference, and audio generation. In **Studio → Sound → Simple**, choose a **Prompt expansion model** before -submitting. The list contains installed, enabled chat models. **Backend default** +submitting. Sound uses the same model picker as Home, with search for longer +model lists and keyboard navigation. The list contains installed, enabled chat +models. You must explicitly choose a model or **Backend default**; opening or +dismissing the picker does not select a model. **Backend default** omits `expansion_model` and uses the audio backend's existing prompt behavior. Sound remembers either choice in this browser, separately from your chat model. If a remembered model disappears or loses chat support, choose again.