fix(ui): use token usage for chat throughput (#12923)

Streaming chunks can contain multiple tokens. Use reported completion
totals for the final end-to-end average and label live rates as estimates.
Count each client tool turn once and include reasoning in the total.

Assisted-by: Codex:gpt-6

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/http/react-ui/e2e/chat-throughput.spec.js

@@ -0,0 +1,48 @@
+import { test, expect } from './coverage-fixtures.js'
+import { chatOf, mockChat, openChat, controlStream } from './chat-fixtures.js'
+
+async function start(page) {
+  await mockChat(page)
+  await controlStream(page)
+  await openChat(page, [chatOf('c1', 'Throughput', 'qwen3-8b', [])])
+  // Only Date is fixed: browser timers and React continue running normally.
+  await page.clock.setFixedTime(new Date('2026-10-26T12:00:00Z'))
+  await page.getByTestId('chat-input').fill('hello')
+  await page.getByTestId('chat-send').click()
+  await expect.poll(() => page.evaluate(() => window.__sse.calls)).toBe(1)
+}
+
+async function atTwoSeconds(page, usage) {
+  await page.clock.setFixedTime(new Date('2026-10-26T12:00:02Z'))
+  await page.evaluate((usage) => {
+    window.__sse.push({ choices: [{ delta: { reasoning: 'thinking about the answer' } }] })
+    window.__sse.push({ choices: [{ delta: { content: 'The answer has several tokens.' } }] })
+    if (usage) window.__sse.push({ choices: [], usage })
+    window.__sse.end()
+  }, usage)
+}
+
+test('chat throughput uses completion usage, including reasoning, over request duration', async ({ page }) => {
+  await start(page)
+  await atTwoSeconds(page, { prompt_tokens: 10, completion_tokens: 100, total_tokens: 110,
+    completion_tokens_details: { reasoning_tokens: 80 } })
+  await expect(page.getByTestId('chat-foot')).toContainText('50 tok/s · end-to-end')
+  await expect(page.getByTestId('chat-foot')).not.toContainText('estimated')
+  await expect(page.getByTestId('chat-foot')).not.toContainText('Peak')
+})
+
+test('chat throughput labels live and missing-usage rates as estimates', async ({ page }) => {
+  await start(page)
+  await page.clock.setFixedTime(new Date('2026-10-26T12:00:01Z'))
+  await page.evaluate(() => window.__sse.push({ choices: [{ delta: { content: '12345678' } }] }))
+  await expect(page.getByTestId('chat-foot')).toContainText('2 tok/s · estimated')
+  await page.clock.setFixedTime(new Date('2026-10-26T12:00:02Z'))
+  await page.evaluate(() => window.__sse.end())
+  await expect(page.getByTestId('chat-foot')).toContainText('1 tok/s · estimated')
+})
+
+test('chat throughput accepts a reported zero completion count', async ({ page }) => {
+  await start(page)
+  await atTwoSeconds(page, { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 })
+  await expect(page.getByTestId('chat-foot')).toContainText('0 tok/s · end-to-end')
+})

core/http/react-ui/public/locales/en/chat.json

@@ -114,7 +114,8 @@
   },
   "tokens": {
     "perSec": "{{count}} tok/s",
-    "peak": "Peak: {{count}} tok/s",
+    "estimatedPerSec": "{{count}} tok/s · estimated",
+    "averagePerSec": "{{count}} tok/s · end-to-end",
     "usage": "{{prompt}}p + {{completion}}c = {{total}}",
     "generating": "generating",
     "ofContext": "{{used}} / {{size}} tokens"

core/http/react-ui/src/hooks/useChat.js

@@ -243,15 +243,14 @@ export function useChat(initialModel = '') {
   const [streamingReasoning, setStreamingReasoning] = useState('')
   const [streamingToolCalls, setStreamingToolCalls] = useState([])
   const [tokensPerSecond, setTokensPerSecond] = useState(null)
-  const [maxTokensPerSecond, setMaxTokensPerSecond] = useState(null)
+  const [throughputIsEstimated, setThroughputIsEstimated] = useState(true)
   // Live progress of a cold load the request is waiting on, so the composer can
   // say "staging to nvidia-thor, 41%" instead of showing an error for a model
   // that is loading exactly as it should.
   const [modelLoading, setModelLoading] = useState(null)
   const abortControllerRef = useRef(null)
   const startTimeRef = useRef(null)
-  const tokenCountRef = useRef(0)
-  const maxTpsRef = useRef(0)
+  const outputCharactersRef = useRef(0)
 
   const activeChat = chats.find(c => c.id === activeChatId) || chats[0]
 
@@ -426,7 +425,7 @@ export function useChat(initialModel = '') {
     setStreamingReasoning('')
     setStreamingToolCalls([])
     setTokensPerSecond(null)
-    setMaxTokensPerSecond(null)
+    setThroughputIsEstimated(true)
   }, [])
 
   const deleteChat = useCallback((chatId) => {
@@ -453,7 +452,7 @@ export function useChat(initialModel = '') {
     setStreamingReasoning('')
     setStreamingToolCalls([])
     setTokensPerSecond(null)
-    setMaxTokensPerSecond(null)
+    setThroughputIsEstimated(true)
     serializedSentRef.current.clear()
     if (!accountId) return
     // Covers every delete still pending before it.
@@ -640,15 +639,18 @@ export function useChat(initialModel = '') {
     setStreamingReasoning('')
     setStreamingToolCalls([])
     setTokensPerSecond(null)
-    setMaxTokensPerSecond(null)
+    setThroughputIsEstimated(true)
     startTimeRef.current = Date.now()
-    tokenCountRef.current = 0
-    maxTpsRef.current = 0
+    outputCharactersRef.current = 0
 
     let usage = {}
+    let completionTokens = 0
+    let completeUsage = true
     const newMessages = [] // Accumulate messages to add to history
 
     if (activeChat.mcpMode && !hasMcpServers) {
+      // Legacy MCP events do not report completion-token usage.
+      completeUsage = false
       // Legacy MCP SSE streaming (custom event types from /v1/mcp/chat/completions)
       try {
         const timeoutId = setTimeout(() => controller.abort(), 300000) // 5 min timeout
@@ -692,7 +694,7 @@ export function useChat(initialModel = '') {
                   hasReasoningFromAPI = true
                   if (eventData.content) {
                     reasoningContent += eventData.content
-                    tokenCountRef.current += Math.ceil(eventData.content.length / 4)
+                    outputCharactersRef.current += eventData.content.length
                     showReasoning(reasoningContent)
                     updateTps()
                   }
@@ -732,7 +734,7 @@ export function useChat(initialModel = '') {
                 case 'assistant':
                   if (eventData.content) {
                     assistantContent += eventData.content
-                    tokenCountRef.current += Math.ceil(eventData.content.length / 4)
+                    outputCharactersRef.current += eventData.content.length
                     // Handle thinking tags if no API reasoning
                     if (!hasReasoningFromAPI) {
                       const { regularContent, thinkingContent } = extractThinking(assistantContent)
@@ -796,6 +798,7 @@ export function useChat(initialModel = '') {
         let currentToolCalls = []
         let finishReason = null
         let fullToolCalls = [] // Tool calls with id for agentic loop
+        let turnCompletionTokens = null
 
         try {
           const response = await fetchWithModelLoadWait(apiUrl(endpoint), {
@@ -832,6 +835,7 @@ export function useChat(initialModel = '') {
 
                 // Handle structured error events
                 if (parsed.error) {
+                  completeUsage = false
                   const errMsg = typeof parsed.error === 'string'
                     ? parsed.error
                     : parsed.error.message || 'Unknown error'
@@ -874,7 +878,7 @@ export function useChat(initialModel = '') {
                 if (delta?.reasoning) {
                   hasReasoningFromAPI = true
                   reasoningContent += delta.reasoning
-                  tokenCountRef.current++
+                  outputCharactersRef.current += delta.reasoning.length
                   showReasoning(reasoningContent)
                   updateTps()
                 }
@@ -911,7 +915,7 @@ export function useChat(initialModel = '') {
 
                 if (delta?.content) {
                   rawContent += delta.content
-                  tokenCountRef.current++
+                  outputCharactersRef.current += delta.content.length
 
                   if (!hasReasoningFromAPI) {
                     if (openThinkTagRegex.test(rawContent) && !closeThinkTagRegex.test(rawContent)) {
@@ -949,6 +953,11 @@ export function useChat(initialModel = '') {
                 }
                 if (parsed?.usage) {
                   usage = parsed.usage
+                  if (Number.isFinite(usage.completion_tokens) && usage.completion_tokens >= 0) {
+                    // Usage is cumulative within a turn, not a delta. Reasoning
+                    // is already included in completion_tokens.
+                    turnCompletionTokens = usage.completion_tokens
+                  }
                 }
               } catch (_e) {
                 // skip malformed JSON
@@ -956,11 +965,15 @@ export function useChat(initialModel = '') {
             }
           }
         } catch (err) {
+          completeUsage = false
           if (err.name !== 'AbortError') {
             rawContent += `\n\nError: ${err.message}`
           }
         }
 
+        if (turnCompletionTokens === null) completeUsage = false
+        else completionTokens += turnCompletionTokens
+
         // Client-side agentic loop: check for client tool calls
         const validToolCalls = fullToolCalls.filter(Boolean)
         const hasClientToolCalls = (
@@ -1055,10 +1068,11 @@ export function useChat(initialModel = '') {
     setStreamingReasoning('')
     setStreamingToolCalls([])
 
-    // Set max tokens/sec badge
-    if (maxTpsRef.current > 0) {
-      setMaxTokensPerSecond(Math.round(maxTpsRef.current * 10) / 10)
-    }
+    // Use the same end-to-end window for every turn, including tool execution.
+    // An interrupted stream may not have reported its final usage.
+    const reported = completeUsage && !controller.signal.aborted
+    updateTps(reported ? completionTokens : undefined)
+    setThroughputIsEstimated(!reported)
 
     // Add messages to history
     if (newMessages.length > 0) {
@@ -1078,14 +1092,14 @@ export function useChat(initialModel = '') {
     }
   }, [activeChat, chats])
 
-  function updateTps() {
+  function updateTps(reportedTokens) {
     const elapsed = (Date.now() - startTimeRef.current) / 1000
     if (elapsed > 0) {
-      const tps = tokenCountRef.current / elapsed
+      // Four characters per token is only a live estimate, independent of
+      // how the backend splits content and reasoning into streaming chunks.
+      const tokens = reportedTokens ?? outputCharactersRef.current / 4
+      const tps = tokens / elapsed
       setTokensPerSecond(Math.round(tps * 10) / 10)
-      if (tps > maxTpsRef.current) {
-        maxTpsRef.current = tps
-      }
     }
   }
 
@@ -1124,7 +1138,7 @@ export function useChat(initialModel = '') {
     streamingReasoning: isActiveStreaming ? streamingReasoning : '',
     streamingToolCalls: isActiveStreaming ? streamingToolCalls : [],
     tokensPerSecond,
-    maxTokensPerSecond,
+    throughputIsEstimated,
     modelLoading: isActiveStreaming ? modelLoading : null,
     addChat,
     forkChat,

core/http/react-ui/src/pages/Chat.jsx

@@ -123,7 +123,7 @@ function ChatPage() {
   const { operations } = useOperations()
   const {
     chats, activeChat, activeChatId, isStreaming, streamingChatId, streamingContent,
-    streamingReasoning, streamingToolCalls, tokensPerSecond, maxTokensPerSecond, modelLoading,
+    streamingReasoning, streamingToolCalls, tokensPerSecond, throughputIsEstimated, modelLoading,
     addChat, forkChat, switchChat, deleteChat, deleteAllChats, renameChat, updateChatSettings,
     sendMessage, stopGeneration, clearHistory, getContextUsagePercent, addMessage,
   } = useChat(urlModel || '')
@@ -1322,10 +1322,10 @@ function ChatPage() {
             <div className="cx-foot" data-testid="chat-foot">
               <span data-warn={(!isStreaming && contextPercent !== null && contextPercent > 90) || undefined}>
                 {isStreaming
-                  ? (tokensPerSecond !== null ? `${t('tokens.perSec', { count: tokensPerSecond })} · ${t('tokens.generating')}` : t('tokens.generating'))
+                  ? (tokensPerSecond !== null ? `${t('tokens.estimatedPerSec', { count: tokensPerSecond })} · ${t('tokens.generating')}` : t('tokens.generating'))
                   : (contextPercent !== null && contextPercent > 90
                     ? t('context.nearlyFull')
-                    : (maxTokensPerSecond !== null ? t('tokens.peak', { count: maxTokensPerSecond }) : ''))}
+                    : (tokensPerSecond !== null ? t(throughputIsEstimated ? 'tokens.estimatedPerSec' : 'tokens.averagePerSec', { count: tokensPerSecond }) : ''))}
               </span>
               <span className="cx-foot__tokens">
                 {activeChat.tokenUsage?.total > 0 && (activeChat.contextSize

core/http/react-ui/tests/unit/chatthroughput.test.mjs

@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: MIT
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import vm from 'node:vm'
+
+// Run the real streaming hook with synchronous hook slots. Browser-only imports
+// and effects are replaced; fetch still returns real ReadableStream responses.
+function mount() {
+  const slots = []
+  let cursor = 0
+  let now = 1000
+  let streams = []
+  const context = vm.createContext({
+    useState(initial) {
+      const i = cursor++
+      if (!(i in slots)) slots[i] = typeof initial === 'function' ? initial() : initial
+      return [slots[i], value => { slots[i] = typeof value === 'function' ? value(slots[i]) : value }]
+    },
+    useRef(initial) {
+      const i = cursor++
+      return slots[i] ??= { current: initial }
+    },
+    useCallback: fn => fn, useEffect() {}, useDebouncedEffect() {},
+    useContext: () => null, RegistryContext: {}, useAuth: () => ({ user: null }),
+    chatsStorageKey: () => 'chats', readChatStore: () => null,
+    effectiveSystemPrompt: text => text, toolAddons: () => [],
+    addonRequestMetadata: () => ({}), chatHeaders: () => ({}), storedResult: () => null,
+    API_CONFIG: { endpoints: { chatCompletions: '/v1/chat/completions' } }, apiUrl: path => path,
+    Date: class extends Date { static now() { return now } },
+    Math, generateId: () => 'chat1', AbortController, TextDecoder, TextDecoderStream, setTimeout, clearTimeout,
+    fetch: async () => {
+      const next = streams.shift()
+      now += next.duration
+      if (next.error) throw next.error
+      return new Response(next.events.map(e => `data: ${JSON.stringify(e)}\n\n`).join('') + 'data: [DONE]\n\n')
+    },
+  })
+  const source = readFileSync(new URL('../../src/hooks/useChat.js', import.meta.url), 'utf8')
+  vm.runInContext(source.replace(/^import .*\n/gm, '').replace('export function useChat', 'function useChat'), context)
+  const render = () => { cursor = 0; return context.useChat('test-model') }
+  return {
+    render,
+    async send(responses, options = {}) {
+      streams = responses
+      await render().sendMessage('hello', [], options)
+      return render()
+    },
+  }
+}
+
+const delta = value => ({ choices: [{ delta: value }] })
+const usage = count => ({ choices: [], usage: { completion_tokens: count } })
+
+test('chat throughput: multi-token chunks use reported totals including reasoning', async () => {
+  const state = await mount().send([{ duration: 2000, events: [
+    delta({ reasoning: 'many reasoning tokens' }), delta({ content: 'many answer tokens' }), usage(100),
+  ] }])
+  assert.equal(state.tokensPerSecond, 50)
+  assert.equal(state.throughputIsEstimated, false)
+})
+
+test('chat throughput: missing usage uses a character estimate independent of chunk boundaries', async () => {
+  for (const content of [['12345678'], ['1', '234', '5678']]) {
+    const state = await mount().send([{ duration: 2000, events: content.map(text => delta({ content: text })) }])
+    assert.equal(state.tokensPerSecond, 1)
+    assert.equal(state.throughputIsEstimated, true)
+  }
+})
+
+test('chat throughput: zero usage is valid and repeated totals are not added', async () => {
+  for (const count of [0, 100]) {
+    const state = await mount().send([{ duration: 2000, events: [delta({ content: 'text' }), usage(count), usage(count)] }])
+    assert.equal(state.tokensPerSecond, count / 2)
+    assert.equal(state.throughputIsEstimated, false)
+  }
+})
+
+test('chat throughput: tool turns sum completion totals across the full elapsed time', async () => {
+  const state = await mount().send([
+    { duration: 2000, events: [delta({ tool_calls: [{ index: 0, id: 't1', type: 'function', function: { name: 'lookup', arguments: '{}' } }] }),
+      { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }, usage(20)] },
+    { duration: 2000, events: [delta({ content: 'done' }), usage(100)] },
+  ], { isClientTool: () => true, executeTool: async () => 'result' })
+  assert.equal(state.tokensPerSecond, 30)
+  assert.equal(state.throughputIsEstimated, false)
+})
+
+test('chat throughput: a later turn without usage cannot reuse an earlier total', async () => {
+  const state = await mount().send([
+    { duration: 2000, events: [delta({ tool_calls: [{ index: 0, id: 't1', type: 'function', function: { name: 'lookup', arguments: '{}' } }] }),
+      { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }, usage(20)] },
+    { duration: 2000, events: [delta({ content: '1234567890123456' })] },
+  ], { isClientTool: () => true, executeTool: async () => 'result' })
+  assert.equal(state.tokensPerSecond, 1)
+  assert.equal(state.throughputIsEstimated, true)
+})
+
+test('chat throughput: failed and aborted requests never show reported throughput', async () => {
+  for (const name of ['Error', 'AbortError']) {
+    const error = new Error('request interrupted')
+    error.name = name
+    const state = await mount().send([{ duration: 2000, error }])
+    assert.equal(state.throughputIsEstimated, true)
+  }
+})
+
+test('chat throughput: zero elapsed time does not produce an infinite rate', async () => {
+  const state = await mount().send([{ duration: 0, events: [delta({ content: 'answer' }), usage(100)] }])
+  assert.equal(state.tokensPerSecond, null)
+})

docs/content/features/chat.md

@@ -46,6 +46,8 @@ The **MCP** chip opens the server and client tool lists. **Canvas** turns code b
 
 Enter sends, Shift+Enter adds a line. Paste an image to attach it. Under the box, the line shows the speed while a reply streams and the token count of the chat.
 
+The live speed is an estimate based on four output characters per token, including reasoning. When the reply finishes and every request reports completion-token usage, the speed uses those totals instead. This **end-to-end** average includes request latency, model loading, prompt processing, and client tool execution across all turns; it is not the backend’s generation-only speed. Reasoning tokens are already part of the reported completion total and are not added again. If usage is missing or the reply is interrupted, the speed remains labeled **estimated**.
+
 The buttons under the box attach an image, an audio file or a document. A PDF is sent as its text layer. A scanned PDF has none, so its first ten pages are attached as images instead, for a model that reads images; a toast says how many pages went along.
 
 The microphone button dictates: click it to record, click again to stop, and the recording goes to `/v1/audio/transcriptions` with the first installed transcription model ([audio to text]({{< relref "audio-to-text.md" >}})). The text lands in the box, so you can read and correct it before sending. Without a transcription model the button is not shown. Recording needs microphone access in the browser, which browsers only grant on `https://` or `localhost`.