diff --git a/core/http/react-ui/e2e/chat-throughput.spec.js b/core/http/react-ui/e2e/chat-throughput.spec.js new file mode 100644 index 000000000..4e1d995e2 --- /dev/null +++ b/core/http/react-ui/e2e/chat-throughput.spec.js @@ -0,0 +1,48 @@ +import { test, expect } from './coverage-fixtures.js' +import { chatOf, mockChat, openChat, controlStream } from './chat-fixtures.js' + +async function start(page) { + await mockChat(page) + await controlStream(page) + await openChat(page, [chatOf('c1', 'Throughput', 'qwen3-8b', [])]) + // Only Date is fixed: browser timers and React continue running normally. + await page.clock.setFixedTime(new Date('2026-10-26T12:00:00Z')) + await page.getByTestId('chat-input').fill('hello') + await page.getByTestId('chat-send').click() + await expect.poll(() => page.evaluate(() => window.__sse.calls)).toBe(1) +} + +async function atTwoSeconds(page, usage) { + await page.clock.setFixedTime(new Date('2026-10-26T12:00:02Z')) + await page.evaluate((usage) => { + window.__sse.push({ choices: [{ delta: { reasoning: 'thinking about the answer' } }] }) + window.__sse.push({ choices: [{ delta: { content: 'The answer has several tokens.' } }] }) + if (usage) window.__sse.push({ choices: [], usage }) + window.__sse.end() + }, usage) +} + +test('chat throughput uses completion usage, including reasoning, over request duration', async ({ page }) => { + await start(page) + await atTwoSeconds(page, { prompt_tokens: 10, completion_tokens: 100, total_tokens: 110, + completion_tokens_details: { reasoning_tokens: 80 } }) + await expect(page.getByTestId('chat-foot')).toContainText('50 tok/s · end-to-end') + await expect(page.getByTestId('chat-foot')).not.toContainText('estimated') + await expect(page.getByTestId('chat-foot')).not.toContainText('Peak') +}) + +test('chat throughput labels live and missing-usage rates as estimates', async ({ page }) => { + await start(page) + await page.clock.setFixedTime(new Date('2026-10-26T12:00:01Z')) + await page.evaluate(() => window.__sse.push({ choices: [{ delta: { content: '12345678' } }] })) + await expect(page.getByTestId('chat-foot')).toContainText('2 tok/s · estimated') + await page.clock.setFixedTime(new Date('2026-10-26T12:00:02Z')) + await page.evaluate(() => window.__sse.end()) + await expect(page.getByTestId('chat-foot')).toContainText('1 tok/s · estimated') +}) + +test('chat throughput accepts a reported zero completion count', async ({ page }) => { + await start(page) + await atTwoSeconds(page, { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 }) + await expect(page.getByTestId('chat-foot')).toContainText('0 tok/s · end-to-end') +}) diff --git a/core/http/react-ui/public/locales/en/chat.json b/core/http/react-ui/public/locales/en/chat.json index b3f4353e1..19f053378 100644 --- a/core/http/react-ui/public/locales/en/chat.json +++ b/core/http/react-ui/public/locales/en/chat.json @@ -114,7 +114,8 @@ }, "tokens": { "perSec": "{{count}} tok/s", - "peak": "Peak: {{count}} tok/s", + "estimatedPerSec": "{{count}} tok/s · estimated", + "averagePerSec": "{{count}} tok/s · end-to-end", "usage": "{{prompt}}p + {{completion}}c = {{total}}", "generating": "generating", "ofContext": "{{used}} / {{size}} tokens" diff --git a/core/http/react-ui/src/hooks/useChat.js b/core/http/react-ui/src/hooks/useChat.js index 8a73ca988..c3775cd45 100644 --- a/core/http/react-ui/src/hooks/useChat.js +++ b/core/http/react-ui/src/hooks/useChat.js @@ -243,15 +243,14 @@ export function useChat(initialModel = '') { const [streamingReasoning, setStreamingReasoning] = useState('') const [streamingToolCalls, setStreamingToolCalls] = useState([]) const [tokensPerSecond, setTokensPerSecond] = useState(null) - const [maxTokensPerSecond, setMaxTokensPerSecond] = useState(null) + const [throughputIsEstimated, setThroughputIsEstimated] = useState(true) // Live progress of a cold load the request is waiting on, so the composer can // say "staging to nvidia-thor, 41%" instead of showing an error for a model // that is loading exactly as it should. const [modelLoading, setModelLoading] = useState(null) const abortControllerRef = useRef(null) const startTimeRef = useRef(null) - const tokenCountRef = useRef(0) - const maxTpsRef = useRef(0) + const outputCharactersRef = useRef(0) const activeChat = chats.find(c => c.id === activeChatId) || chats[0] @@ -426,7 +425,7 @@ export function useChat(initialModel = '') { setStreamingReasoning('') setStreamingToolCalls([]) setTokensPerSecond(null) - setMaxTokensPerSecond(null) + setThroughputIsEstimated(true) }, []) const deleteChat = useCallback((chatId) => { @@ -453,7 +452,7 @@ export function useChat(initialModel = '') { setStreamingReasoning('') setStreamingToolCalls([]) setTokensPerSecond(null) - setMaxTokensPerSecond(null) + setThroughputIsEstimated(true) serializedSentRef.current.clear() if (!accountId) return // Covers every delete still pending before it. @@ -640,15 +639,18 @@ export function useChat(initialModel = '') { setStreamingReasoning('') setStreamingToolCalls([]) setTokensPerSecond(null) - setMaxTokensPerSecond(null) + setThroughputIsEstimated(true) startTimeRef.current = Date.now() - tokenCountRef.current = 0 - maxTpsRef.current = 0 + outputCharactersRef.current = 0 let usage = {} + let completionTokens = 0 + let completeUsage = true const newMessages = [] // Accumulate messages to add to history if (activeChat.mcpMode && !hasMcpServers) { + // Legacy MCP events do not report completion-token usage. + completeUsage = false // Legacy MCP SSE streaming (custom event types from /v1/mcp/chat/completions) try { const timeoutId = setTimeout(() => controller.abort(), 300000) // 5 min timeout @@ -692,7 +694,7 @@ export function useChat(initialModel = '') { hasReasoningFromAPI = true if (eventData.content) { reasoningContent += eventData.content - tokenCountRef.current += Math.ceil(eventData.content.length / 4) + outputCharactersRef.current += eventData.content.length showReasoning(reasoningContent) updateTps() } @@ -732,7 +734,7 @@ export function useChat(initialModel = '') { case 'assistant': if (eventData.content) { assistantContent += eventData.content - tokenCountRef.current += Math.ceil(eventData.content.length / 4) + outputCharactersRef.current += eventData.content.length // Handle thinking tags if no API reasoning if (!hasReasoningFromAPI) { const { regularContent, thinkingContent } = extractThinking(assistantContent) @@ -796,6 +798,7 @@ export function useChat(initialModel = '') { let currentToolCalls = [] let finishReason = null let fullToolCalls = [] // Tool calls with id for agentic loop + let turnCompletionTokens = null try { const response = await fetchWithModelLoadWait(apiUrl(endpoint), { @@ -832,6 +835,7 @@ export function useChat(initialModel = '') { // Handle structured error events if (parsed.error) { + completeUsage = false const errMsg = typeof parsed.error === 'string' ? parsed.error : parsed.error.message || 'Unknown error' @@ -874,7 +878,7 @@ export function useChat(initialModel = '') { if (delta?.reasoning) { hasReasoningFromAPI = true reasoningContent += delta.reasoning - tokenCountRef.current++ + outputCharactersRef.current += delta.reasoning.length showReasoning(reasoningContent) updateTps() } @@ -911,7 +915,7 @@ export function useChat(initialModel = '') { if (delta?.content) { rawContent += delta.content - tokenCountRef.current++ + outputCharactersRef.current += delta.content.length if (!hasReasoningFromAPI) { if (openThinkTagRegex.test(rawContent) && !closeThinkTagRegex.test(rawContent)) { @@ -949,6 +953,11 @@ export function useChat(initialModel = '') { } if (parsed?.usage) { usage = parsed.usage + if (Number.isFinite(usage.completion_tokens) && usage.completion_tokens >= 0) { + // Usage is cumulative within a turn, not a delta. Reasoning + // is already included in completion_tokens. + turnCompletionTokens = usage.completion_tokens + } } } catch (_e) { // skip malformed JSON @@ -956,11 +965,15 @@ export function useChat(initialModel = '') { } } } catch (err) { + completeUsage = false if (err.name !== 'AbortError') { rawContent += `\n\nError: ${err.message}` } } + if (turnCompletionTokens === null) completeUsage = false + else completionTokens += turnCompletionTokens + // Client-side agentic loop: check for client tool calls const validToolCalls = fullToolCalls.filter(Boolean) const hasClientToolCalls = ( @@ -1055,10 +1068,11 @@ export function useChat(initialModel = '') { setStreamingReasoning('') setStreamingToolCalls([]) - // Set max tokens/sec badge - if (maxTpsRef.current > 0) { - setMaxTokensPerSecond(Math.round(maxTpsRef.current * 10) / 10) - } + // Use the same end-to-end window for every turn, including tool execution. + // An interrupted stream may not have reported its final usage. + const reported = completeUsage && !controller.signal.aborted + updateTps(reported ? completionTokens : undefined) + setThroughputIsEstimated(!reported) // Add messages to history if (newMessages.length > 0) { @@ -1078,14 +1092,14 @@ export function useChat(initialModel = '') { } }, [activeChat, chats]) - function updateTps() { + function updateTps(reportedTokens) { const elapsed = (Date.now() - startTimeRef.current) / 1000 if (elapsed > 0) { - const tps = tokenCountRef.current / elapsed + // Four characters per token is only a live estimate, independent of + // how the backend splits content and reasoning into streaming chunks. + const tokens = reportedTokens ?? outputCharactersRef.current / 4 + const tps = tokens / elapsed setTokensPerSecond(Math.round(tps * 10) / 10) - if (tps > maxTpsRef.current) { - maxTpsRef.current = tps - } } } @@ -1124,7 +1138,7 @@ export function useChat(initialModel = '') { streamingReasoning: isActiveStreaming ? streamingReasoning : '', streamingToolCalls: isActiveStreaming ? streamingToolCalls : [], tokensPerSecond, - maxTokensPerSecond, + throughputIsEstimated, modelLoading: isActiveStreaming ? modelLoading : null, addChat, forkChat, diff --git a/core/http/react-ui/src/pages/Chat.jsx b/core/http/react-ui/src/pages/Chat.jsx index 3ada91edd..3ca1c6b6e 100644 --- a/core/http/react-ui/src/pages/Chat.jsx +++ b/core/http/react-ui/src/pages/Chat.jsx @@ -123,7 +123,7 @@ function ChatPage() { const { operations } = useOperations() const { chats, activeChat, activeChatId, isStreaming, streamingChatId, streamingContent, - streamingReasoning, streamingToolCalls, tokensPerSecond, maxTokensPerSecond, modelLoading, + streamingReasoning, streamingToolCalls, tokensPerSecond, throughputIsEstimated, modelLoading, addChat, forkChat, switchChat, deleteChat, deleteAllChats, renameChat, updateChatSettings, sendMessage, stopGeneration, clearHistory, getContextUsagePercent, addMessage, } = useChat(urlModel || '') @@ -1322,10 +1322,10 @@ function ChatPage() {
90) || undefined}> {isStreaming - ? (tokensPerSecond !== null ? `${t('tokens.perSec', { count: tokensPerSecond })} · ${t('tokens.generating')}` : t('tokens.generating')) + ? (tokensPerSecond !== null ? `${t('tokens.estimatedPerSec', { count: tokensPerSecond })} · ${t('tokens.generating')}` : t('tokens.generating')) : (contextPercent !== null && contextPercent > 90 ? t('context.nearlyFull') - : (maxTokensPerSecond !== null ? t('tokens.peak', { count: maxTokensPerSecond }) : ''))} + : (tokensPerSecond !== null ? t(throughputIsEstimated ? 'tokens.estimatedPerSec' : 'tokens.averagePerSec', { count: tokensPerSecond }) : ''))} {activeChat.tokenUsage?.total > 0 && (activeChat.contextSize diff --git a/core/http/react-ui/tests/unit/chatthroughput.test.mjs b/core/http/react-ui/tests/unit/chatthroughput.test.mjs new file mode 100644 index 000000000..a4b9d2cde --- /dev/null +++ b/core/http/react-ui/tests/unit/chatthroughput.test.mjs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: MIT +import test from 'node:test' +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import vm from 'node:vm' + +// Run the real streaming hook with synchronous hook slots. Browser-only imports +// and effects are replaced; fetch still returns real ReadableStream responses. +function mount() { + const slots = [] + let cursor = 0 + let now = 1000 + let streams = [] + const context = vm.createContext({ + useState(initial) { + const i = cursor++ + if (!(i in slots)) slots[i] = typeof initial === 'function' ? initial() : initial + return [slots[i], value => { slots[i] = typeof value === 'function' ? value(slots[i]) : value }] + }, + useRef(initial) { + const i = cursor++ + return slots[i] ??= { current: initial } + }, + useCallback: fn => fn, useEffect() {}, useDebouncedEffect() {}, + useContext: () => null, RegistryContext: {}, useAuth: () => ({ user: null }), + chatsStorageKey: () => 'chats', readChatStore: () => null, + effectiveSystemPrompt: text => text, toolAddons: () => [], + addonRequestMetadata: () => ({}), chatHeaders: () => ({}), storedResult: () => null, + API_CONFIG: { endpoints: { chatCompletions: '/v1/chat/completions' } }, apiUrl: path => path, + Date: class extends Date { static now() { return now } }, + Math, generateId: () => 'chat1', AbortController, TextDecoder, TextDecoderStream, setTimeout, clearTimeout, + fetch: async () => { + const next = streams.shift() + now += next.duration + if (next.error) throw next.error + return new Response(next.events.map(e => `data: ${JSON.stringify(e)}\n\n`).join('') + 'data: [DONE]\n\n') + }, + }) + const source = readFileSync(new URL('../../src/hooks/useChat.js', import.meta.url), 'utf8') + vm.runInContext(source.replace(/^import .*\n/gm, '').replace('export function useChat', 'function useChat'), context) + const render = () => { cursor = 0; return context.useChat('test-model') } + return { + render, + async send(responses, options = {}) { + streams = responses + await render().sendMessage('hello', [], options) + return render() + }, + } +} + +const delta = value => ({ choices: [{ delta: value }] }) +const usage = count => ({ choices: [], usage: { completion_tokens: count } }) + +test('chat throughput: multi-token chunks use reported totals including reasoning', async () => { + const state = await mount().send([{ duration: 2000, events: [ + delta({ reasoning: 'many reasoning tokens' }), delta({ content: 'many answer tokens' }), usage(100), + ] }]) + assert.equal(state.tokensPerSecond, 50) + assert.equal(state.throughputIsEstimated, false) +}) + +test('chat throughput: missing usage uses a character estimate independent of chunk boundaries', async () => { + for (const content of [['12345678'], ['1', '234', '5678']]) { + const state = await mount().send([{ duration: 2000, events: content.map(text => delta({ content: text })) }]) + assert.equal(state.tokensPerSecond, 1) + assert.equal(state.throughputIsEstimated, true) + } +}) + +test('chat throughput: zero usage is valid and repeated totals are not added', async () => { + for (const count of [0, 100]) { + const state = await mount().send([{ duration: 2000, events: [delta({ content: 'text' }), usage(count), usage(count)] }]) + assert.equal(state.tokensPerSecond, count / 2) + assert.equal(state.throughputIsEstimated, false) + } +}) + +test('chat throughput: tool turns sum completion totals across the full elapsed time', async () => { + const state = await mount().send([ + { duration: 2000, events: [delta({ tool_calls: [{ index: 0, id: 't1', type: 'function', function: { name: 'lookup', arguments: '{}' } }] }), + { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }, usage(20)] }, + { duration: 2000, events: [delta({ content: 'done' }), usage(100)] }, + ], { isClientTool: () => true, executeTool: async () => 'result' }) + assert.equal(state.tokensPerSecond, 30) + assert.equal(state.throughputIsEstimated, false) +}) + +test('chat throughput: a later turn without usage cannot reuse an earlier total', async () => { + const state = await mount().send([ + { duration: 2000, events: [delta({ tool_calls: [{ index: 0, id: 't1', type: 'function', function: { name: 'lookup', arguments: '{}' } }] }), + { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }, usage(20)] }, + { duration: 2000, events: [delta({ content: '1234567890123456' })] }, + ], { isClientTool: () => true, executeTool: async () => 'result' }) + assert.equal(state.tokensPerSecond, 1) + assert.equal(state.throughputIsEstimated, true) +}) + +test('chat throughput: failed and aborted requests never show reported throughput', async () => { + for (const name of ['Error', 'AbortError']) { + const error = new Error('request interrupted') + error.name = name + const state = await mount().send([{ duration: 2000, error }]) + assert.equal(state.throughputIsEstimated, true) + } +}) + +test('chat throughput: zero elapsed time does not produce an infinite rate', async () => { + const state = await mount().send([{ duration: 0, events: [delta({ content: 'answer' }), usage(100)] }]) + assert.equal(state.tokensPerSecond, null) +}) diff --git a/docs/content/features/chat.md b/docs/content/features/chat.md index bdc30d495..c5ee96717 100644 --- a/docs/content/features/chat.md +++ b/docs/content/features/chat.md @@ -46,6 +46,8 @@ The **MCP** chip opens the server and client tool lists. **Canvas** turns code b Enter sends, Shift+Enter adds a line. Paste an image to attach it. Under the box, the line shows the speed while a reply streams and the token count of the chat. +The live speed is an estimate based on four output characters per token, including reasoning. When the reply finishes and every request reports completion-token usage, the speed uses those totals instead. This **end-to-end** average includes request latency, model loading, prompt processing, and client tool execution across all turns; it is not the backend’s generation-only speed. Reasoning tokens are already part of the reported completion total and are not added again. If usage is missing or the reply is interrupted, the speed remains labeled **estimated**. + The buttons under the box attach an image, an audio file or a document. A PDF is sent as its text layer. A scanned PDF has none, so its first ten pages are attached as images instead, for a model that reads images; a toast says how many pages went along. The microphone button dictates: click it to record, click again to stop, and the recording goes to `/v1/audio/transcriptions` with the first installed transcription model ([audio to text]({{< relref "audio-to-text.md" >}})). The text lands in the box, so you can read and correct it before sending. Without a transcription model the button is not shown. Recording needs microphone access in the browser, which browsers only grant on `https://` or `localhost`.