fix(ci): shard single-arch backend matrix under GitHub's 256-job limit

GitHub Actions refuses to instantiate a matrix that would generate more
than 256 jobs. It does so silently: the job hangs forever at "Waiting for
pending jobs" and the whole run is marked `failure` while every other job
stays green. This is exactly what happened on the v4.6.1 tag build
(run 28786533892): the single-arch build matrix had grown to 268 entries,
so `backend-jobs-singlearch` (and its downstream merge) never produced a
single job, and the release build "failed" with no failing job to point at.

The single-arch list is the one that grows unbounded as backends are added,
so shard it across a fixed number of matrix jobs (SINGLEARCH_SHARDS=4,
~67 entries each today, headroom to ~1020 backends). Each merge shard
`needs:` only its matching build shard, preserving the "merge waits only on
its own build" property that keeps slow CUDA/ROCm builds from gating
multi-arch manifest assembly.

changed-backends.js now emits per-shard matrix/has-* outputs and throws
loudly if a shard ever reaches the 256 limit (telling the maintainer to
bump SINGLEARCH_SHARDS and add matching job blocks) instead of letting
GitHub drop the overflow silently. backend.yml and backend_pr.yml define
the four build + four merge shard jobs; multi-arch and darwin groups are
untouched.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:claude-opus-4-8 [Claude Code]

.github/workflows/backend.yml

@@ -32,16 +32,30 @@ jobs:
     if: github.repository == 'mudler/LocalAI'
     runs-on: ubuntu-latest
     outputs:
-      matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }}
       matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }}
       matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }}
       merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }}
-      merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }}
-      has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }}
       has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }}
       has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }}
       has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }}
-      has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }}
+      # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to
+      # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js).
+      matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }}
+      merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }}
+      has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }}
+      has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }}
+      matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }}
+      merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }}
+      has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }}
+      has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }}
+      matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }}
+      merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }}
+      has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }}
+      has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }}
+      matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }}
+      merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }}
+      has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }}
+      has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v7
@@ -109,9 +123,9 @@ jobs:
   # take their full ~6h cold without blocking manifest assembly for the
   # multi-arch backends whose per-arch digests would otherwise sit untagged
   # on quay long enough to be GC'd.
-  backend-jobs-singlearch:
+  backend-jobs-singlearch-1:
     needs: generate-matrix
-    if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true'
+    if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true'
     uses: ./.github/workflows/backend_build.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -138,7 +152,100 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }}
+
+  backend-jobs-singlearch-2:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }}
+
+  backend-jobs-singlearch-3:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }}
+
+  backend-jobs-singlearch-4:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }}
 
   # Apply tags to per-arch digests via `imagetools create`. Split into two
   # jobs that mirror the build split so each merge waits ONLY on its
@@ -174,10 +281,12 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }}
 
-  backend-merge-jobs-singlearch:
-    needs: [generate-matrix, backend-jobs-singlearch]
-    # See note on backend-merge-jobs-multiarch above for !cancelled().
-    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }}
+  # One merge shard per build shard: backend-merge-jobs-singlearch-<n> needs only
+  # backend-jobs-singlearch-<n>, preserving the "merge waits only on its own
+  # build" property while staying under the 256-jobs-per-matrix limit.
+  backend-merge-jobs-singlearch-1:
+    needs: [generate-matrix, backend-jobs-singlearch-1]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }}
     uses: ./.github/workflows/backend_merge.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -189,7 +298,55 @@ jobs:
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }}
+
+  backend-merge-jobs-singlearch-2:
+    needs: [generate-matrix, backend-jobs-singlearch-2]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }}
+
+  backend-merge-jobs-singlearch-3:
+    needs: [generate-matrix, backend-jobs-singlearch-3]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }}
+
+  backend-merge-jobs-singlearch-4:
+    needs: [generate-matrix, backend-jobs-singlearch-4]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }}
 
   backend-jobs-darwin:
     needs: generate-matrix

.github/workflows/backend_pr.yml

@@ -11,16 +11,30 @@ jobs:
   generate-matrix:
     runs-on: ubuntu-latest
     outputs:
-      matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }}
       matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }}
       matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }}
       merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }}
-      merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }}
-      has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }}
       has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }}
       has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }}
       has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }}
-      has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }}
+      # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to
+      # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js).
+      matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }}
+      merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }}
+      has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }}
+      has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }}
+      matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }}
+      merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }}
+      has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }}
+      has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }}
+      matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }}
+      merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }}
+      has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }}
+      has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }}
+      matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }}
+      merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }}
+      has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }}
+      has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v7
@@ -71,10 +85,10 @@ jobs:
       fail-fast: true
       max-parallel: 8
       matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-multiarch']) }}
-  backend-jobs-singlearch:
+  backend-jobs-singlearch-1:
     needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true'
     uses: ./.github/workflows/backend_build.yml
-    if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true'
     with:
       tag-latest: ${{ matrix.tag-latest }}
       tag-suffix: ${{ matrix.tag-suffix }}
@@ -98,7 +112,94 @@ jobs:
     strategy:
       fail-fast: true
       max-parallel: 8
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }}
+
+  backend-jobs-singlearch-2:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: true
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }}
+
+  backend-jobs-singlearch-3:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: true
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }}
+
+  backend-jobs-singlearch-4:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: true
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }}
   backend-merge-jobs-multiarch:
     needs: [generate-matrix, backend-jobs-multiarch]
     # backend_merge.yml's push-side steps are all gated on
@@ -118,9 +219,9 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }}
 
-  backend-merge-jobs-singlearch:
-    needs: [generate-matrix, backend-jobs-singlearch]
-    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }}
+  backend-merge-jobs-singlearch-1:
+    needs: [generate-matrix, backend-jobs-singlearch-1]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }}
     uses: ./.github/workflows/backend_merge.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -130,7 +231,49 @@ jobs:
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }}
+
+  backend-merge-jobs-singlearch-2:
+    needs: [generate-matrix, backend-jobs-singlearch-2]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }}
+
+  backend-merge-jobs-singlearch-3:
+    needs: [generate-matrix, backend-jobs-singlearch-3]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }}
+
+  backend-merge-jobs-singlearch-4:
+    needs: [generate-matrix, backend-jobs-singlearch-4]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }}
   backend-jobs-darwin:
     needs: generate-matrix
     uses: ./.github/workflows/backend_build_darwin.yml

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=fdb1db877c526ec90f668eca1b858da5dba85560
+LLAMA_VERSION?=2da668617612d2df773f966e3b0ee22dc2beef7b
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/run.sh

@@ -36,6 +36,12 @@ else
 	if [ -d "$CURDIR/lib/rocblas/library" ]; then
 		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
 	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
 fi
 
 # If there is a lib/ld.so, use it

backend/cpp/turboquant/run.sh

@@ -34,6 +34,12 @@ else
 	if [ -d "$CURDIR/lib/rocblas/library" ]; then
 		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
 	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
 fi
 
 # If there is a lib/ld.so, use it

backend/go/acestep-cpp/CMakeLists.txt

@@ -25,7 +25,7 @@ target_include_directories(goacestepcpp PRIVATE ${ACESTEP_DIR}/src ${ACESTEP_DIR
 target_include_directories(goacestepcpp SYSTEM PRIVATE ${ACESTEP_DIR}/ggml/include)
 
 # Link GPU backends if available (mirrors link_ggml_backends macro)
-foreach(backend blas cuda metal vulkan)
+foreach(backend blas cuda hip metal vulkan)
     if(TARGET ggml-${backend})
         target_link_libraries(goacestepcpp PRIVATE ggml-${backend})
         string(TOUPPER ${backend} BACKEND_UPPER)

backend/go/acestep-cpp/Makefile

@@ -24,7 +24,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=9a26976a8c8cf5af0afcdd04463cf8ba91e96a54
+CRISPASR_VERSION?=09df654e304947f7521e1f52992ceacccf03c300
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/omnivoice-cpp/CMakeLists.txt

@@ -30,7 +30,7 @@ target_include_directories(gomnivoicecpp PRIVATE ${OMNIVOICE_DIR}/src)
 target_include_directories(gomnivoicecpp SYSTEM PRIVATE ${OMNIVOICE_DIR}/ggml/include)
 
 # Link GPU backends if the upstream ggml created them.
-foreach(backend blas cuda metal vulkan sycl)
+foreach(backend blas cuda hip metal vulkan sycl)
     if(TARGET ggml-${backend})
         target_link_libraries(gomnivoicecpp PRIVATE ggml-${backend})
         if(backend STREQUAL "cuda")

backend/go/omnivoice-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # omnivoice.cpp version
 OMNIVOICE_REPO?=https://github.com/ServeurpersoCom/omnivoice.cpp
-OMNIVOICE_VERSION?=0f37401bebe9b20c0160a888e592108fc1d17607
+OMNIVOICE_VERSION?=daedb763fd442e0916eb130a479fdd74947291c0
 SO_TARGET?=libgomnivoicecpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
@@ -24,7 +24,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/go/qwen3-tts-cpp/CMakeLists.txt

@@ -30,7 +30,7 @@ target_include_directories(goqwen3ttscpp PRIVATE ${QWENTTS_DIR}/src)
 target_include_directories(goqwen3ttscpp SYSTEM PRIVATE ${QWENTTS_DIR}/ggml/include)
 
 # Link GPU backends if the upstream ggml created them.
-foreach(backend blas cuda metal vulkan sycl)
+foreach(backend blas cuda hip metal vulkan sycl)
     if(TARGET ggml-${backend})
         target_link_libraries(goqwen3ttscpp PRIVATE ggml-${backend})
         if(backend STREQUAL "cuda")

backend/go/qwen3-tts-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # qwentts.cpp version
 QWEN3TTS_REPO?=https://github.com/ServeurpersoCom/qwentts.cpp
-QWEN3TTS_CPP_VERSION?=9dbe7ea26a01b30fccb117ae5e86807c1dc23d42
+QWEN3TTS_CPP_VERSION?=73fe0c67bbf0898ba2999535e0680a02a7f8537d
 SO_TARGET?=libgoqwen3ttscpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
@@ -24,7 +24,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/go/vibevoice-cpp/CMakeLists.txt

@@ -50,7 +50,7 @@ target_include_directories(govibevoicecpp SYSTEM PRIVATE ${VIBEVOICE_DIR}/third_
 # Link GPU backends if available — vibevoice's own CMake already links
 # these to the libvibevoice STATIC library, but we re-link them on the
 # MODULE so resolved symbols include all backend kernels.
-foreach(backend blas cuda metal vulkan)
+foreach(backend blas cuda hip metal vulkan)
     if(TARGET ggml-${backend})
         target_link_libraries(govibevoicecpp PRIVATE ggml-${backend})
         string(TOUPPER ${backend} BACKEND_UPPER)

backend/go/vibevoice-cpp/Makefile

@@ -29,7 +29,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DVIBEVOICE_GGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON -DVIBEVOICE_GGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/python/ace-step/requirements-cpu.txt

@@ -4,7 +4,7 @@ torchaudio
 torchvision
 
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio
 matplotlib>=3.7.5

backend/python/ace-step/requirements-cublas12.txt

@@ -4,7 +4,7 @@ torchaudio
 torchvision
 
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio>=6.5.1
 matplotlib>=3.7.5

backend/python/ace-step/requirements-cublas13.txt

@@ -4,7 +4,7 @@ torchaudio
 torchvision
 
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio>=6.5.1
 matplotlib>=3.7.5

backend/python/ace-step/requirements-hipblas.txt

@@ -1,10 +1,10 @@
 --extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.12.0+cpu
+torch==2.10.0+rocm7.0
 torchaudio
 torchvision
 
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio>=6.5.1
 matplotlib>=3.7.5

backend/python/ace-step/requirements-intel.txt

@@ -4,7 +4,7 @@ torchaudio
 torchvision
 
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio
 matplotlib>=3.7.5

backend/python/ace-step/requirements-l4t13.txt

@@ -3,7 +3,7 @@ torch
 torchaudio
 torchvision
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio>=6.5.1
 matplotlib>=3.7.5

backend/python/ace-step/requirements-mps.txt

@@ -3,7 +3,7 @@ torchaudio
 torchvision
 
 # Core dependencies
-transformers>=5.13.0,<5.14.0
+transformers>=4.51.0,<4.58.0
 diffusers
 gradio
 matplotlib>=3.7.5

backend/python/rfdetr/requirements-cpu.txt

@@ -3,5 +3,5 @@ opencv-python
 accelerate
 peft
 inference
-torch==2.12.0+cu130
+torch==2.7.1
 optimum-quanto

backend/python/rfdetr/requirements-cublas12.txt

@@ -1,4 +1,4 @@
-torch==2.12.0+cu130
+torch==2.7.1
 rfdetr
 opencv-python
 accelerate

backend/python/rfdetr/requirements-cublas13.txt

@@ -1,5 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch==2.12.0+cu130
+torch==2.9.1
 rfdetr
 opencv-python
 accelerate

backend/python/rfdetr/requirements-hipblas.txt

@@ -1,5 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.12.0+cu130
+torch==2.10.0+rocm7.0
 torchvision==0.25.0+rocm7.0
 rfdetr
 opencv-python

backend/python/rfdetr/requirements-mps.txt

@@ -1,4 +1,4 @@
-torch==2.12.0+cu130
+torch==2.7.1
 rfdetr
 opencv-python
 accelerate

backend/python/sglang/requirements-cpu.txt

@@ -1,6 +1,6 @@
 --extra-index-url https://download.pytorch.org/whl/cpu
 accelerate
-torch==2.12.0+cpu
+torch==2.9.0
 torchvision
 torchaudio
 transformers

backend/python/sglang/requirements-cublas12.txt

@@ -6,7 +6,7 @@
 # for cublas12 so uv consults this index alongside PyPI.
 --extra-index-url https://download.pytorch.org/whl/cu128
 accelerate
-torch==2.12.0+cpu
+torch==2.9.1
 torchvision
 torchaudio
 transformers

backend/python/trl/requirements-cpu.txt

@@ -1,9 +1,9 @@
 --extra-index-url https://download.pytorch.org/whl/cpu
-torch==2.12.0+cpu
+torch==2.10.0
 trl
 peft
 datasets>=3.0.0
-transformers>=5.13.0
+transformers>=4.56.2
 accelerate>=1.4.0
 huggingface-hub>=1.3.0
 sentencepiece

backend/python/trl/requirements-cublas12.txt

@@ -1,8 +1,8 @@
-torch==2.12.0+cpu
+torch==2.10.0
 trl
 peft
 datasets>=3.0.0
-transformers>=5.13.0
+transformers>=4.56.2
 accelerate>=1.4.0
 huggingface-hub>=1.3.0
 sentencepiece

backend/python/trl/requirements-cublas13.txt

@@ -1,8 +1,8 @@
-torch==2.12.0+cpu
+torch==2.10.0
 trl
 peft
 datasets>=3.0.0
-transformers>=5.13.0
+transformers>=4.56.2
 accelerate>=1.4.0
 huggingface-hub>=1.3.0
 sentencepiece

backend/python/trl/requirements-mps.txt

@@ -1,8 +1,8 @@
-torch==2.12.0+cpu
+torch==2.10.0
 trl
 peft
 datasets>=3.0.0
-transformers>=5.13.0
+transformers>=4.56.2
 accelerate>=1.4.0
 huggingface-hub>=1.3.0
 sentencepiece

backend/python/vllm-omni/requirements-cublas12.txt

@@ -1,4 +1,4 @@
 accelerate
-torch==2.12.0+cu130
+torch==2.7.0
 transformers
 bitsandbytes

backend/python/vllm/install.sh

@@ -119,7 +119,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux
     # vllm pin (requirements-cublas13-after.txt, bumped independently against
     # vllm/vllm) until vllm-metal supports a newer vLLM.
-    VLLM_METAL_VERSION="v0.3.0.dev20260701212152"
+    VLLM_METAL_VERSION="v0.3.0.dev20260704102955"
 
     # The coupled vLLM source version is whatever this vllm-metal release builds
     # against -- it declares it in its own installer as `vllm_v=`. Derive it from

cmd/launcher/internal/launcher.go

@@ -207,21 +207,7 @@ func (l *Launcher) StartLocalAI() error {
 	}
 
 	// Build command arguments
-	dataPath := l.GetDataPath()
-	args := []string{
-		"run",
-		"--models-path", l.config.ModelsPath,
-		"--backends-path", l.config.BackendsPath,
-		"--address", l.config.Address,
-		"--log-level", l.config.LogLevel,
-		// Keep persistent data and dynamic config under the launcher's data
-		// directory (~/.localai) rather than letting the server resolve them
-		// to ${basepath}/{data,configuration}. ${basepath} expands to the
-		// launcher process's CWD (often the user's home root), which puts
-		// ~/data and ~/configuration outside ~/.localai. See #10610.
-		"--data-path", filepath.Join(dataPath, "data"),
-		"--localai-config-dir", filepath.Join(dataPath, "configuration"),
-	}
+	args := l.BuildRunArgs()
 
 	l.localaiCmd = exec.CommandContext(l.ctx, binaryPath, args...)
 
@@ -406,6 +392,32 @@ func (l *Launcher) GetWebUIURL() string {
 	return address
 }
 
+// BuildRunArgs assembles the argument list passed to `local-ai run`.
+//
+// Storage paths are anchored to the launcher's data directory instead of the
+// server's own defaults. The server resolves data/config to ${basepath}
+// (the launcher process CWD, often the user's home root) and generated-content
+// /uploads to shared /tmp paths. On a shared /tmp (macOS routes /tmp to
+// /private/tmp for every user) the first account to run LocalAI creates
+// /tmp/generated with 0750 perms, so any other account then fails startup with
+// "mkdir /tmp/generated/content: permission denied". Keeping every writable
+// path under the per-user data directory avoids both the misplacement (#10610)
+// and the cross-user /tmp collision.
+func (l *Launcher) BuildRunArgs() []string {
+	dataPath := l.GetDataPath()
+	return []string{
+		"run",
+		"--models-path", l.config.ModelsPath,
+		"--backends-path", l.config.BackendsPath,
+		"--address", l.config.Address,
+		"--log-level", l.config.LogLevel,
+		"--data-path", filepath.Join(dataPath, "data"),
+		"--localai-config-dir", filepath.Join(dataPath, "configuration"),
+		"--generated-content-path", filepath.Join(dataPath, "generated"),
+		"--upload-path", filepath.Join(dataPath, "uploads"),
+	}
+}
+
 // GetDataPath returns the path where LocalAI data and logs are stored
 func (l *Launcher) GetDataPath() string {
 	// LocalAI typically stores data in the current working directory or a models directory

cmd/launcher/internal/launcher_test.go

@@ -149,6 +149,41 @@ var _ = Describe("Launcher", func() {
 		})
 	})
 
+	Describe("BuildRunArgs", func() {
+		// Regression for the macOS "mkdir /tmp/generated/content: permission denied"
+		// startup failure: the launcher must redirect generated-content and upload
+		// paths under its own data directory instead of letting the server fall back
+		// to the shared /tmp defaults, which collide across users on a shared /tmp.
+		It("should keep generated-content and upload paths under the data directory", func() {
+			config := launcherInstance.GetConfig()
+			config.ModelsPath = filepath.Join(tempDir, "models")
+			launcherInstance.SetConfig(config)
+
+			dataPath := launcherInstance.GetDataPath()
+			args := launcherInstance.BuildRunArgs()
+
+			assertFlagValue := func(flag, expected string) {
+				idx := -1
+				for i, a := range args {
+					if a == flag {
+						idx = i
+						break
+					}
+				}
+				Expect(idx).To(BeNumerically(">=", 0), "expected %s to be present in run args", flag)
+				Expect(idx+1).To(BeNumerically("<", len(args)), "expected a value after %s", flag)
+				Expect(args[idx+1]).To(Equal(expected))
+			}
+
+			assertFlagValue("--generated-content-path", filepath.Join(dataPath, "generated"))
+			assertFlagValue("--upload-path", filepath.Join(dataPath, "uploads"))
+			// The bug was the server resolving these to shared /tmp paths.
+			for _, a := range args {
+				Expect(a).ToNot(HavePrefix("/tmp/"), "run args must not reference shared /tmp paths, got %s", a)
+			}
+		})
+	})
+
 	Describe("Logs", func() {
 		It("should return empty logs initially", func() {
 			logs := launcherInstance.GetLogs()

cmd/local-ai/main.go

@@ -57,10 +57,17 @@ For documentation and support:
 		),
 		kong.UsageOnError(),
 		kong.Vars{
-			"basepath":  kong.ExpandPath("."),
-			"galleries": `[{"name":"localai", "url":"github:mudler/LocalAI/gallery/index.yaml@master"}]`,
-			"backends":  `[{"name":"localai", "url":"github:mudler/LocalAI/backend/index.yaml@master"}]`,
-			"version":   internal.PrintableVersion(),
+			"basepath": kong.ExpandPath("."),
+			// Per-user temp locations for ephemeral writable content. A fixed
+			// shared name under /tmp collides across accounts on multi-user hosts
+			// (notably macOS, where /tmp is the shared /private/tmp for everyone),
+			// failing startup with "mkdir /tmp/generated/content: permission
+			// denied". See cli.DefaultGeneratedContentPath.
+			"generatedcontentpath": cli.DefaultGeneratedContentPath(),
+			"uploadpath":           cli.DefaultUploadPath(),
+			"galleries":            `[{"name":"localai", "url":"github:mudler/LocalAI/gallery/index.yaml@master"}]`,
+			"backends":             `[{"name":"localai", "url":"github:mudler/LocalAI/backend/index.yaml@master"}]`,
+			"version":              internal.PrintableVersion(),
 		},
 	)
 	ctx, err := k.Parse(os.Args[1:])

core/backend/llm.go

@@ -47,6 +47,28 @@ func needsThinkingProbe(c *config.ModelConfig) bool {
 			c.ReasoningConfig.DisableReasoningTagPrefill == nil)
 }
 
+// persistProbedReasoning writes the post-probe reasoning slots (and media
+// marker) from probed back into the loader's persisted config for modelName,
+// skipping any reasoning slot the probe was not actually allowed to fill.
+// persistDisableReasoning/persistDisableTagPrefill must be snapshotted from
+// probed's reasoning slots *before* the probe ran: a slot that already
+// carried a value at that point was populated by request-time
+// ApplyReasoningEffort, not by backend detection, and persisting it would
+// masquerade as an operator's explicit reasoning.disable (see #10622).
+func persistProbedReasoning(cl *config.ModelConfigLoader, modelName string, probed *config.ModelConfig, persistDisableReasoning, persistDisableTagPrefill bool) {
+	cl.UpdateModelConfig(modelName, func(cfg *config.ModelConfig) {
+		if persistDisableReasoning {
+			cfg.ReasoningConfig.DisableReasoning = probed.ReasoningConfig.DisableReasoning
+		}
+		if persistDisableTagPrefill {
+			cfg.ReasoningConfig.DisableReasoningTagPrefill = probed.ReasoningConfig.DisableReasoningTagPrefill
+		}
+		if probed.MediaMarker != "" {
+			cfg.MediaMarker = probed.MediaMarker
+		}
+	})
+}
+
 // HasChatDeltaContent returns true if any chat delta carries content or reasoning text.
 // Used to decide whether to prefer C++ autoparser deltas over Go-side tag extraction.
 func (t TokenUsage) HasChatDeltaContent() bool {
@@ -127,15 +149,19 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 	needsMarkerProbe := c.MediaMarker == ""
 	if shouldProbeThinking || needsMarkerProbe {
 		modelOpts := grpcModelOpts(*c, o.SystemState.Model.ModelsPath)
+		// DetectThinkingSupportFromBackend only fills reasoning slots that are
+		// still nil, so a slot that already carries a value here was populated by
+		// request-time ApplyReasoningEffort (e.g. a `reasoning_effort: none`
+		// default), not by backend detection. Persisting such a request-scoped
+		// value would masquerade as an operator's explicit reasoning.disable and
+		// permanently defeat future per-request reasoning_effort overrides
+		// (see #10622). Only persist the slots the probe is actually allowed to
+		// fill.
+		persistDisableReasoning := c.ReasoningConfig.DisableReasoning == nil
+		persistDisableTagPrefill := c.ReasoningConfig.DisableReasoningTagPrefill == nil
 		config.DetectThinkingSupportFromBackend(ctx, c, inferenceModel, modelOpts)
 		// Update the config in the loader so it persists for future requests
-		cl.UpdateModelConfig(c.Name, func(cfg *config.ModelConfig) {
-			cfg.ReasoningConfig.DisableReasoning = c.ReasoningConfig.DisableReasoning
-			cfg.ReasoningConfig.DisableReasoningTagPrefill = c.ReasoningConfig.DisableReasoningTagPrefill
-			if c.MediaMarker != "" {
-				cfg.MediaMarker = c.MediaMarker
-			}
-		})
+		persistProbedReasoning(cl, c.Name, c, persistDisableReasoning, persistDisableTagPrefill)
 	}
 
 	var protoMessages []*proto.Message

core/backend/llm_probe_test.go

@@ -1,6 +1,8 @@
 package backend
 
 import (
+	"os"
+
 	"github.com/mudler/LocalAI/core/config"
 
 	"github.com/gpustack/gguf-parser-go/util/ptr"
@@ -27,3 +29,90 @@ var _ = Describe("thinking probe gating", func() {
 		Expect(needsThinkingProbe(cfg)).To(BeFalse())
 	})
 })
+
+var _ = Describe("persistProbedReasoning", func() {
+	const modelName = "probe-test"
+
+	// newLoaderWithConfig seeds a ModelConfigLoader with a single model config
+	// parsed from yamlBody, mirroring how the loader is populated from disk.
+	newLoaderWithConfig := func(yamlBody string) *config.ModelConfigLoader {
+		tmp, err := os.CreateTemp("", "persist-probed-reasoning-*.yaml")
+		Expect(err).ToNot(HaveOccurred())
+		defer func() { _ = os.Remove(tmp.Name()) }()
+
+		_, err = tmp.WriteString(yamlBody)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(tmp.Close()).To(Succeed())
+
+		cl := config.NewModelConfigLoader("")
+		Expect(cl.ReadModelConfig(tmp.Name())).To(Succeed())
+		return cl
+	}
+
+	It("persists a reasoning slot the probe was allowed to fill (was nil beforehand)", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		probed.ReasoningConfig.DisableReasoning = ptr.To(false) // backend detected: supports thinking
+		probed.ReasoningConfig.DisableReasoningTagPrefill = ptr.To(true)
+
+		persistProbedReasoning(cl, modelName, probed, true, true)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoning).To(BeFalse())
+		Expect(cfg.ReasoningConfig.DisableReasoningTagPrefill).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoningTagPrefill).To(BeTrue())
+	})
+
+	It("does not persist a slot that already carried a request-scoped value before the probe ran", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		// Simulates ApplyReasoningEffort("none") having set this on the
+		// request-scoped copy before the probe ran - not a genuine backend
+		// detection, so it must never reach the persisted config (#10622).
+		probed.ReasoningConfig.DisableReasoning = ptr.To(true)
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).To(BeNil())
+		Expect(cfg.ReasoningConfig.DisableReasoningTagPrefill).To(BeNil())
+	})
+
+	It("preserves an operator's explicit persisted disable when the guard is false", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\nreasoning:\n  disable: true\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		// Even if the request-scoped copy ends up holding a different value,
+		// persistDisableReasoning=false must keep the operator's own setting.
+		probed.ReasoningConfig.DisableReasoning = ptr.To(false)
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoning).To(BeTrue())
+	})
+
+	It("persists the media marker regardless of the reasoning guards", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		probed.MediaMarker = "<__media__>"
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.MediaMarker).To(Equal("<__media__>"))
+	})
+})

core/backend/options.go

@@ -223,13 +223,24 @@ func EffectiveContextSize(c config.ModelConfig) int {
 	return DefaultContextSize
 }
 
+// localGPU resolves the device that will run the model, for single-pass batch
+// sizing. It is a package var so tests inject a deterministic device; production
+// reads config.LocalGPU, whose detection is sync.Once-cached in xsysinfo — so the
+// per-request call from the router's prompt trimmer (modelTokenTrim) stays cheap.
+var localGPU = config.LocalGPU
+
 // EffectiveBatchSize is the single-decode batch the backend will run with.
 // Score, embedding and rerank all process the whole input in one pass: score
 // decodes prompt+candidate (asserts n_tokens <= n_batch), and embedding/rerank
-// pool over the full sequence in one physical batch (n_ubatch). So the batch
-// is sized to the context — anything that fits the context fits one pass,
+// pool over the full sequence in one physical batch (n_ubatch). Ideally the batch
+// covers the whole context so any input that fits the context fits one pass,
 // avoiding both the GGML_ASSERT crash and the "input is too large to process"
-// error. Explicit `batch:` always wins.
+// error — BUT a full ctx-sized n_ubatch makes the per-device CUDA compute buffer
+// multi-GiB (it scales ~ n_ubatch * n_ctx and can't be split across GPUs), so a
+// large-context embedding model aborts on load with free VRAM to spare (#10485).
+// So we cap the batch to the largest that fits the per-device VRAM headroom; an
+// input longer than that cap is the accepted tradeoff (it can't be pooled in one
+// pass, but the load no longer OOMs). Explicit `batch:` always wins.
 func EffectiveBatchSize(c config.ModelConfig) int {
 	if c.Batch != 0 {
 		return c.Batch
@@ -238,7 +249,7 @@ func EffectiveBatchSize(c config.ModelConfig) int {
 		c.HasUsecases(config.FLAG_EMBEDDINGS) ||
 		c.HasUsecases(config.FLAG_RERANK)
 	if ctx := EffectiveContextSize(c); singlePass && ctx > DefaultBatchSize {
-		return ctx
+		return config.SinglePassBatchForContext(localGPU(), ctx)
 	}
 	return DefaultBatchSize
 }

core/backend/options_internal_test.go

@@ -103,6 +103,19 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 	threads := 1
 	ctx := 4096
 
+	// The single-pass batch is now VRAM-aware, so inject a deterministic GPU with
+	// ample per-device VRAM: at these small contexts the compute buffer fits
+	// easily, so EffectiveBatchSize returns the full context (the pre-#10485
+	// behaviour these cases assert). Without injection the value would depend on
+	// the CI host's real (often unknown) VRAM.
+	const gib = uint64(1) << 30
+	var origLocalGPU func() config.GPU
+	BeforeEach(func() {
+		origLocalGPU = localGPU
+		localGPU = func() config.GPU { return config.GPU{VRAM: 119 * gib} }
+	})
+	AfterEach(func() { localGPU = origLocalGPU })
+
 	It("defaults to 512 for an ordinary model", func() {
 		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
 		opts := grpcModelOpts(cfg, "/tmp/models")
@@ -162,6 +175,61 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 	})
 })
 
+// Guards the VRAM-aware cap on the single-pass (embedding/score/rerank) batch:
+// a large context must not turn n_ubatch into a multi-GiB compute buffer that
+// aborts the load on a device with free VRAM (issue #10485). The GPU is injected
+// via the localGPU package var so the cap is deterministic without a real device.
+var _ = Describe("EffectiveBatchSize VRAM cap", func() {
+	const gib = uint64(1) << 30
+	embeddings := config.FLAG_EMBEDDINGS
+	threads := 1
+
+	var origLocalGPU func() config.GPU
+	BeforeEach(func() { origLocalGPU = localGPU })
+	AfterEach(func() { localGPU = origLocalGPU })
+
+	singlePassCfg := func(ctx int) config.ModelConfig {
+		return config.ModelConfig{
+			Threads:       &threads,
+			LLMConfig:     config.LLMConfig{ContextSize: &ctx},
+			KnownUsecases: &embeddings,
+		}
+	}
+
+	It("caps a large embedding context to a batch below the context but at least the default", func() {
+		// Reproduces qwen3-embedding-4b: context 40960 on a modest 20 GiB card.
+		// Full-context n_ubatch=40960 aborts; the cap must fit the VRAM headroom.
+		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
+		batch := EffectiveBatchSize(singlePassCfg(40960))
+		Expect(batch).To(BeNumerically(">=", DefaultBatchSize))
+		Expect(batch).To(BeNumerically("<", 40960))
+	})
+
+	It("keeps an explicit batch even with a large context and small VRAM", func() {
+		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
+		cfg := singlePassCfg(40960)
+		cfg.Batch = 512
+		Expect(EffectiveBatchSize(cfg)).To(Equal(512))
+	})
+
+	It("returns the full context when per-device VRAM is unknown", func() {
+		// Unknown VRAM (CPU / detection gap) preserves the original single-pass
+		// behavior: batch follows context. The VRAM cap is a downward safety that
+		// only engages when the per-device ceiling is known — clamping here would
+		// re-break single-pass pooling and over-trim inputs, with no OOM benefit on
+		// CPU where the compute buffer lives in system RAM.
+		localGPU = func() config.GPU { return config.GPU{VRAM: 0} }
+		Expect(EffectiveBatchSize(singlePassCfg(40960))).To(Equal(40960))
+	})
+
+	It("returns the default batch for a non-single-pass model regardless of VRAM", func() {
+		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
+		ctx := 40960
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+		Expect(EffectiveBatchSize(cfg)).To(Equal(DefaultBatchSize))
+	})
+})
+
 // Guards the generic chat_template_kwargs forwarding: the model config map plus any
 // per-request metadata overrides are merged, coerced, and serialised into the
 // backend metadata blob that llama.cpp reads. Client metadata also overrides the

core/cli/chat/session.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"slices"
 	"strings"
 )
 
@@ -58,7 +59,7 @@ func (s *chatSession) Clear() {
 }
 
 func (s *chatSession) SwitchModel(model string) error {
-	if !modelExists(s.models, model) {
+	if !slices.Contains(s.models, model) {
 		return fmt.Errorf("model %q is not available. Use /models to see installed models", model)
 	}
 	s.model = model
@@ -103,18 +104,9 @@ Then start a chat session:
 		b.WriteString("multiple models are available; choose one with --model:\n")
 		b.WriteString(formatChatModelList(models, ""))
 		return "", errors.New(b.String())
-	case !modelExists(models, requested):
+	case !slices.Contains(models, requested):
 		return "", fmt.Errorf("model %q is not available. Use `local-ai models list` and `local-ai models install <model>`, or pass an installed model with --model", requested)
 	default:
 		return requested, nil
 	}
 }
-
-func modelExists(models []string, name string) bool {
-	for _, model := range models {
-		if model == name {
-			return true
-		}
-	}
-	return false
-}

core/cli/run.go

@@ -35,8 +35,8 @@ type RunCMD struct {
 	BackendsPath                 string        `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
 	BackendsSystemPath           string        `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends used for inferencing" group:"backends"`
 	ModelsPath                   string        `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
-	GeneratedContentPath         string        `env:"LOCALAI_GENERATED_CONTENT_PATH,GENERATED_CONTENT_PATH" type:"path" default:"/tmp/generated/content" help:"Location for generated content (e.g. images, audio, videos)" group:"storage"`
-	UploadPath                   string        `env:"LOCALAI_UPLOAD_PATH,UPLOAD_PATH" type:"path" default:"/tmp/localai/upload" help:"Path to store uploads from files api" group:"storage"`
+	GeneratedContentPath         string        `env:"LOCALAI_GENERATED_CONTENT_PATH,GENERATED_CONTENT_PATH" type:"path" default:"${generatedcontentpath}" help:"Location for generated content (e.g. images, audio, videos)" group:"storage"`
+	UploadPath                   string        `env:"LOCALAI_UPLOAD_PATH,UPLOAD_PATH" type:"path" default:"${uploadpath}" help:"Path to store uploads from files api" group:"storage"`
 	DataPath                     string        `env:"LOCALAI_DATA_PATH" type:"path" default:"${basepath}/data" help:"Path for persistent data (collectiondb, agent state, tasks, jobs). Separates mutable data from configuration" group:"storage"`
 	LocalaiConfigDir             string        `env:"LOCALAI_CONFIG_DIR" type:"path" default:"${basepath}/configuration" help:"Directory for dynamic loading of certain configuration files (currently api_keys.json and external_backends.json)" group:"storage"`
 	LocalaiConfigDirPollInterval time.Duration `env:"LOCALAI_CONFIG_DIR_POLL_INTERVAL" help:"Typically the config path picks up changes automatically, but if your system has broken fsnotify events, set this to an interval to poll the LocalAI Config Dir (example: 1m)" group:"storage"`
@@ -186,6 +186,31 @@ type RunCMD struct {
 	PIIDefaultDetectors []string `env:"LOCALAI_PII_DEFAULT_DETECTORS" help:"Instance-wide default PII/secret detector model names applied to any PII-enabled model (chiefly cloud-proxy / MITM models) that names no pii.detectors of its own. Comma-separated, e.g. privacy-filter-nemotron,secret-filter. Takes precedence over the value persisted via the Middleware UI." group:"middleware"`
 }
 
+// userScopedTempDir returns a temp directory namespaced to the current user.
+//
+// The generated-content and upload directories are ephemeral, so they live
+// under the OS temp dir - but a fixed shared name like /tmp/generated is a trap
+// on any multi-user host. macOS routes /tmp to the shared /private/tmp for every
+// account, so whichever user starts LocalAI first creates the parent with 0750
+// perms and every other account then fails startup with
+// "mkdir /tmp/generated/content: permission denied" (the same happens on Linux
+// once a stale root-owned /tmp/generated is left behind). Scoping to the current
+// UID gives each account its own tree so they never collide.
+func userScopedTempDir() string {
+	return filepath.Join(os.TempDir(), fmt.Sprintf("localai-%d", os.Getuid()))
+}
+
+// DefaultGeneratedContentPath returns the default location for backend-generated
+// content (images, audio, videos).
+func DefaultGeneratedContentPath() string {
+	return filepath.Join(userScopedTempDir(), "generated", "content")
+}
+
+// DefaultUploadPath returns the default location for uploads from the files API.
+func DefaultUploadPath() string {
+	return filepath.Join(userScopedTempDir(), "upload")
+}
+
 func (r *RunCMD) Run(ctx *cliContext.Context) error {
 	warnDeprecatedFlags()
 

core/cli/run_paths_test.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Regression for the startup failure observed when a second OS account (or a
+// leftover root-owned directory) already created the shared /tmp locations:
+//
+//	unable to create ImageDir: "mkdir /tmp/generated/content: permission denied"
+//
+// The historical defaults (/tmp/generated/content and /tmp/localai/upload) are
+// shared across every user of a machine. On macOS /tmp is routed to the shared
+// /private/tmp for all accounts, so the first account to run LocalAI creates the
+// parent with 0750 perms and locks everyone else out. The defaults must instead
+// be scoped to the current user so unrelated accounts never collide.
+var _ = Describe("default writable paths", func() {
+	userScope := fmt.Sprintf("localai-%d", os.Getuid())
+
+	Describe("DefaultGeneratedContentPath", func() {
+		It("is scoped to the current user under the OS temp dir", func() {
+			p := DefaultGeneratedContentPath()
+			Expect(p).To(HavePrefix(os.TempDir()))
+			Expect(p).To(ContainSubstring(userScope))
+			Expect(p).To(HaveSuffix(filepath.Join("generated", "content")))
+		})
+
+		It("is not the historical shared path", func() {
+			Expect(DefaultGeneratedContentPath()).ToNot(Equal("/tmp/generated/content"))
+		})
+	})
+
+	Describe("DefaultUploadPath", func() {
+		It("is scoped to the current user under the OS temp dir", func() {
+			p := DefaultUploadPath()
+			Expect(p).To(HavePrefix(os.TempDir()))
+			Expect(p).To(ContainSubstring(userScope))
+			Expect(p).To(HaveSuffix("upload"))
+		})
+
+		It("is not the historical shared path", func() {
+			Expect(DefaultUploadPath()).ToNot(Equal("/tmp/localai/upload"))
+		})
+	})
+})

core/config/context_fit.go

@@ -0,0 +1,144 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/xsysinfo"
+	"github.com/mudler/xlog"
+)
+
+// contextFitHeadroomDivisor reserves a slice of per-device VRAM as headroom when
+// deciding whether an auto-derived context fits. The gguf-parser footprint
+// already covers weights + KV + compute buffer, but a live load also pays for
+// allocator fragmentation, the CUDA/HIP context, and whatever else shares the
+// card, so we require the estimate to leave at least 1/divisor of the device
+// free. /5 (~20% headroom) mirrors the SWA full-cache gate's margin.
+const contextFitHeadroomDivisor = 5
+
+// contextFitCandidates is the descending set of context windows tried when the
+// DefaultAutoContextSize cap itself does not fit per-device VRAM. Only the rare
+// big-model-on-tiny-card case reaches this walk; it is capped at the base
+// choice and floored at DefaultContextSize, and returns the first (largest)
+// candidate that fits.
+var contextFitCandidates = []int{8192, 6144, 4096}
+
+// perDeviceVRAM reports the smallest per-GPU VRAM ceiling in bytes (0 = unknown
+// or no GPU). It is a package var so tests can inject a deterministic value —
+// detection does a live GPU probe. Per-device (not summed) is the right budget:
+// with all layers offloaded to a single device the whole footprint must fit that
+// one card, and a multi-GPU host is bounded by its smallest card. This mirrors
+// localGPU's use of MinPerGPUVRAM in hardware_defaults.go.
+var perDeviceVRAM = func() uint64 {
+	v, _ := xsysinfo.MinPerGPUVRAM()
+	return v
+}
+
+// estimateContextVRAM returns the estimated per-device VRAM footprint (bytes) of
+// running f fully offloaded at ctx tokens — weights + KV cache + compute buffer.
+// It returns 0 when it cannot produce an estimate (nil file, no tensors, or a
+// parser panic), which the caller treats as "cannot confirm a smaller fit" and
+// so keeps the conservative cap rather than clamping on a bogus number. It is a
+// package var so tests can stub it (a fabricated GGUF carries no tensors and
+// estimates to ~0).
+var estimateContextVRAM = func(f *gguf.GGUFFile, ctx int) (footprint uint64) {
+	if f == nil {
+		return 0
+	}
+	if ctx <= 0 {
+		ctx = DefaultContextSize
+	}
+	// The gguf-parser estimator panics on degenerate / partially-parsed GGUFs;
+	// treat any failure as "unknown" so config loading never crashes on a model
+	// the parser mis-handles.
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Debug("[context_fit] per-device VRAM estimate failed; treating as unknown", "error", r)
+			footprint = 0
+		}
+	}()
+	// Offload all layers (LocalAI's DefaultNGPULayers default; the estimator
+	// clamps to the model's block count) so the estimate reflects a fully
+	// GPU-resident model. NonUMA is the discrete-GPU figure (larger than the UMA
+	// one), which keeps the fit check conservative on unified-memory hosts — they
+	// have ample memory to clear it anyway.
+	est := f.EstimateLLaMACppRun(
+		gguf.WithLLaMACppContextSize(int32(ctx)),
+		gguf.WithLLaMACppOffloadLayers(uint64(DefaultNGPULayers)),
+	)
+	sum := est.Summarize(true, 0, 0)
+	if len(sum.Items) == 0 {
+		return 0
+	}
+	var total uint64
+	for _, v := range sum.Items[0].VRAMs {
+		total += uint64(v.NonUMA)
+	}
+	return total
+}
+
+// contextFitsVRAM reports whether an estimated footprint fits a per-device VRAM
+// ceiling with headroom (VRAM must exceed the footprint by ~1/divisor). Unknown
+// inputs (0) are treated as "cannot confirm" so a detection or estimate gap does
+// not clamp the context.
+func contextFitsVRAM(footprint, vram uint64) bool {
+	if footprint == 0 || vram == 0 {
+		return false
+	}
+	return vram >= footprint+footprint/contextFitHeadroomDivisor
+}
+
+// autoContextSize picks the default context to use for f when the user did not
+// set context_size. The choice is deliberately conservative, NOT
+// VRAM-maximizing:
+//
+//  1. Base cap: min(trainedMax, DefaultAutoContextSize). A small model keeps its
+//     trained window; a long-context model (128k / 256k / 1M) is capped so its
+//     KV cache does not default to a size no consumer GPU can hold. This applies
+//     always, including CPU / unknown-VRAM hosts.
+//  2. VRAM is only a downward safety: when a per-device VRAM ceiling IS detected
+//     and even the base cap would not fit it (with headroom), step down through
+//     contextFitCandidates to the largest window that fits, floored at
+//     DefaultContextSize. When VRAM is unknown we skip this — the base cap is
+//     already safe and we must not regress CPU / detection-gap hosts.
+//
+// trainedMax <= 0 means the estimate yielded nothing usable; the caller keeps
+// its existing DefaultContextSize fallback in that case, so this is only called
+// with a positive trainedMax.
+func autoContextSize(f *gguf.GGUFFile, trainedMax int) int {
+	chosen := trainedMax
+	if chosen > DefaultAutoContextSize {
+		chosen = DefaultAutoContextSize
+	}
+
+	vram := perDeviceVRAM()
+	if vram == 0 {
+		// No per-device VRAM detected (CPU-only, unified memory reporting nothing,
+		// or a detection gap). The bug is GPU OOM-on-load, so with no GPU budget to
+		// reason about we must not clamp — the base cap already bounds long-context
+		// models.
+		return chosen
+	}
+
+	if contextFitsVRAM(estimateContextVRAM(f, chosen), vram) {
+		return chosen
+	}
+
+	// The base cap does not fit this card. Walk candidates downward and take the
+	// largest that fits, never below DefaultContextSize.
+	for _, cand := range contextFitCandidates {
+		if cand > chosen || cand < DefaultContextSize {
+			continue
+		}
+		if contextFitsVRAM(estimateContextVRAM(f, cand), vram) {
+			xlog.Debug("[context_fit] capped auto context to fit per-device VRAM",
+				"context", cand, "base_cap", chosen, "vram_gib", vram>>30)
+			return cand
+		}
+	}
+
+	// Nothing fit (an unusually large model on a tiny card): fall back to the
+	// floor. The backend still clamps n_gpu_layers to what fits, so a partial
+	// offload can keep the model loadable rather than aborting outright.
+	xlog.Debug("[context_fit] no candidate context fit per-device VRAM; using floor",
+		"context", DefaultContextSize, "base_cap", chosen, "vram_gib", vram>>30)
+	return DefaultContextSize
+}

core/config/context_fit_internal_test.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// These specs exercise the auto-derived default context. The detection seams
+// (perDeviceVRAM, estimateContextVRAM) are package vars so a deterministic VRAM
+// ceiling and footprint can be injected without a real GPU or model file — the
+// same pattern hardware_defaults_internal_test.go uses for localGPU.
+var _ = Describe("Auto-derived default context (VRAM-aware cap)", func() {
+	const gib = uint64(1) << 30
+
+	var (
+		origVRAM     func() uint64
+		origEstimate func(f *gguf.GGUFFile, ctx int) uint64
+	)
+
+	BeforeEach(func() {
+		origVRAM = perDeviceVRAM
+		origEstimate = estimateContextVRAM
+	})
+	AfterEach(func() {
+		perDeviceVRAM = origVRAM
+		estimateContextVRAM = origEstimate
+	})
+
+	Context("autoContextSize", func() {
+		It("caps a long-context model at DefaultAutoContextSize when VRAM is ample", func() {
+			// 1M-context model on an 80 GiB card: we do NOT chase the trained max,
+			// we keep the conservative 8k cap (users opt into more via context_size).
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib } // trivially fits
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+
+		It("keeps a small model's trained window instead of inflating it", func() {
+			// trained 4096 < 8192: min() keeps 4096, it is not raised to the cap.
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib }
+			Expect(autoContextSize(nil, 4096)).To(Equal(4096))
+		})
+
+		It("steps below the cap when even 8k would not fit a tiny card", func() {
+			// A large model on a 2 GiB card where the 8k footprint overflows but a
+			// smaller context fits: choose the largest that fits, never below the
+			// floor. Footprint grows with context so the walk finds a fit.
+			perDeviceVRAM = func() uint64 { return 2 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, ctx int) uint64 {
+				return gib + uint64(ctx)*100000
+			}
+			chosen := autoContextSize(nil, 1048576)
+			Expect(chosen).To(BeNumerically("<", DefaultAutoContextSize))
+			Expect(chosen).To(BeNumerically(">=", DefaultContextSize))
+			// The chosen context's footprint must actually fit the card with headroom.
+			Expect(contextFitsVRAM(estimateContextVRAM(nil, chosen), 2*gib)).To(BeTrue())
+		})
+
+		It("falls back to the floor when nothing fits", func() {
+			// Even DefaultContextSize does not fit: return the floor and let the
+			// backend clamp n_gpu_layers to what it can (partial offload) rather
+			// than defaulting to a window guaranteed to abort.
+			perDeviceVRAM = func() uint64 { return 1 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 100 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultContextSize))
+		})
+
+		It("does not clamp when per-device VRAM is unknown", func() {
+			// CPU-only / detection gap: no GPU budget to reason about, so we must
+			// not regress — keep the conservative base cap regardless of estimate.
+			perDeviceVRAM = func() uint64 { return 0 }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 999 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+	})
+
+	Context("guessGGUFFromFile", func() {
+		It("never overrides an explicitly configured context_size", func() {
+			// A fabricated GGUF is enough: the context branch is skipped entirely
+			// when the user pinned context_size, so the estimate is never consulted.
+			explicit := 262144
+			cfg := &ModelConfig{LLMConfig: LLMConfig{ContextSize: &explicit}}
+			f := &gguf.GGUFFile{
+				Header: gguf.GGUFHeader{
+					MetadataKV: gguf.GGUFMetadataKVs{
+						{
+							Key:       "general.architecture",
+							ValueType: gguf.GGUFMetadataValueTypeString,
+							Value:     "llama",
+						},
+					},
+				},
+			}
+			guessGGUFFromFile(cfg, f, 0)
+			Expect(cfg.ContextSize).ToNot(BeNil())
+			Expect(*cfg.ContextSize).To(Equal(262144))
+		})
+	})
+})

core/config/defaults.go

@@ -18,6 +18,18 @@ const (
 	// safe default beats a tiny, surprising window that truncates real prompts.
 	DefaultContextSize = 4096
 
+	// DefaultAutoContextSize caps the context we auto-derive from a GGUF when the
+	// user did not set context_size. The GGUF importer used to default a model's
+	// context to its full trained window (n_ctx_train). For long-context models
+	// (128k / 256k / 1M) that KV cache cannot fit a consumer GPU and the backend
+	// aborts on load (exitCode=-1) even though the model file is fine. So instead
+	// of shooting for the trained max, we keep a modest default: a small model
+	// (trained < this) keeps its trained window, while a long-context model caps
+	// here. Users who want the full window raise context_size explicitly. This is
+	// a conservative default, not a VRAM-maximizing one — VRAM is only used to
+	// step further DOWN when even this cap would not fit (see context_fit.go).
+	DefaultAutoContextSize = 8192
+
 	// DefaultNGPULayers means "offload all layers"; the backend (fit_params)
 	// clamps to what actually fits in device memory.
 	DefaultNGPULayers = 99999999

core/config/gguf.go

@@ -28,9 +28,14 @@ func reservedNonChatModel(cfg *ModelConfig) bool {
 
 func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
 	if defaultCtx == 0 && cfg.ContextSize == nil {
-		ctxSize := f.EstimateLLaMACppRun().ContextSize
-		if ctxSize > 0 {
-			cSize := int(ctxSize)
+		// trainedMax is the model's full trained context window (n_ctx_train).
+		// Defaulting a model to it unbounded is what OOMs long-context models at
+		// load: a 128k / 256k / 1M KV cache cannot fit a consumer GPU and the
+		// backend aborts (exitCode=-1). autoContextSize instead caps to a modest
+		// default and only steps below it when detected per-device VRAM demands.
+		trainedMax := int(f.EstimateLLaMACppRun().ContextSize)
+		if trainedMax > 0 {
+			cSize := autoContextSize(f, trainedMax)
 			cfg.ContextSize = &cSize
 		} else {
 			defaultCtx = DefaultContextSize

core/config/hardware_defaults.go

@@ -149,6 +149,51 @@ func largeContextForDevice(g GPU, ctx int) bool {
 	return extra > g.VRAM/blackwellBatchHeadroomDivisor
 }
 
+// SinglePassBatchForContext caps the physical batch (n_batch / n_ubatch) for a
+// single-pass load — embedding, score and rerank all decode/pool the whole input
+// in ONE physical batch, so they want a batch >= the input length to avoid the
+// GGML_ASSERT(n_tokens <= n_batch) abort and the "input is too large to process"
+// error. The naive choice is batch == context, but n_ubatch == context turns the
+// per-device CUDA compute buffer (which scales ~ n_ubatch * n_ctx and is NOT
+// split across GPUs) into multi-GiB of scratch that must fit on a SINGLE card, so
+// a large-context embedding model aborts on load (exitCode=-1) even with plenty
+// of free VRAM — the same #10485 root cause the Blackwell batch boost guards
+// against, which the single-pass path previously bypassed entirely.
+//
+// So instead of the full context we return the LARGEST batch whose compute buffer
+// fits the per-device VRAM headroom (VRAM / blackwellBatchHeadroomDivisor),
+// clamped to [DefaultPhysicalBatch, ctx]. The tradeoff: an input longer than the
+// returned cap can no longer be pooled in a single pass — but a batch that OOMs
+// the device processes nothing at all.
+//
+// g.VRAM must be the PER-DEVICE ceiling (smallest device on a multi-GPU host).
+// VRAM 0 (unknown — CPU-only or a detection gap) returns the full context,
+// preserving the original single-pass behavior (batch follows context): the cap
+// is a DOWNWARD safety that only engages when the per-device ceiling is known.
+// Returning a smaller batch on unknown VRAM would re-break single-pass pooling
+// (n_tokens > n_batch) and over-trim score/embed/rerank inputs, with no OOM
+// benefit on CPU where the buffer lives in system RAM.
+func SinglePassBatchForContext(g GPU, ctx int) int {
+	if ctx <= DefaultPhysicalBatch {
+		return DefaultPhysicalBatch
+	}
+	if g.VRAM == 0 {
+		return ctx
+	}
+	perBatchCell := uint64(ctx) * computeBufferBytesPerCell
+	if perBatchCell == 0 {
+		return DefaultPhysicalBatch
+	}
+	batchCap := int(g.VRAM / blackwellBatchHeadroomDivisor / perBatchCell)
+	if batchCap < DefaultPhysicalBatch {
+		return DefaultPhysicalBatch
+	}
+	if batchCap > ctx {
+		return ctx
+	}
+	return batchCap
+}
+
 // IsManagedPhysicalBatch reports whether n is a value PhysicalBatch assigns.
 // Callers that re-tune a value chosen by an upstream host (the distributed
 // router correcting the frontend's guess) use this to avoid clobbering an
@@ -254,6 +299,14 @@ var localGPU = func() GPU {
 	}
 }
 
+// LocalGPU exposes the locally-detected device descriptor to other packages
+// (e.g. core/backend's single-pass batch sizing) so they resolve the same
+// per-device VRAM this package's heuristics reason about. It goes through the
+// injectable localGPU var, so a config-package test seam also affects callers.
+func LocalGPU() GPU {
+	return localGPU()
+}
+
 // ApplyHardwareDefaults fills ModelConfig values that depend on the target GPU
 // and were left unset by the user. Currently: a larger physical batch on
 // Blackwell. Explicit config always wins (we only touch zero values).

core/config/hardware_defaults_internal_test.go

@@ -46,3 +46,41 @@ var _ = Describe("SetDefaults hardware defaults (single-instance)", func() {
 		Expect(cfg.Batch).To(Equal(1024))
 	})
 })
+
+// SinglePassBatchForContext is the VRAM-aware cap for the single-pass
+// (embedding/score/rerank) batch — the compute buffer scales ~ n_ubatch * n_ctx
+// and must fit a single device, so a large context can't take the full context
+// as its batch (issue #10485).
+var _ = Describe("SinglePassBatchForContext", func() {
+	const gib = uint64(1) << 30
+
+	It("returns the default when the context is at or below the default batch", func() {
+		Expect(SinglePassBatchForContext(GPU{VRAM: 119 * gib}, DefaultPhysicalBatch)).To(Equal(DefaultPhysicalBatch))
+		Expect(SinglePassBatchForContext(GPU{VRAM: 119 * gib}, 256)).To(Equal(DefaultPhysicalBatch))
+	})
+
+	It("returns the full context when the compute buffer fits ample VRAM", func() {
+		// 4096 ctx on 119 GiB: the compute buffer is tiny, so the batch covers
+		// the whole context (single-pass pooling in one physical batch).
+		Expect(SinglePassBatchForContext(GPU{VRAM: 119 * gib}, 4096)).To(Equal(4096))
+	})
+
+	It("caps below the context when a large context would overflow the VRAM headroom", func() {
+		batch := SinglePassBatchForContext(GPU{VRAM: 20 * gib}, 40960)
+		Expect(batch).To(BeNumerically(">=", DefaultPhysicalBatch))
+		Expect(batch).To(BeNumerically("<", 40960))
+		// The compute buffer for the capped batch must fit VRAM/headroom.
+		Expect(uint64(batch) * 40960 * computeBufferBytesPerCell).To(BeNumerically("<=", (20*gib)/blackwellBatchHeadroomDivisor))
+	})
+
+	It("never caps below the default batch even when VRAM is very tight", func() {
+		Expect(SinglePassBatchForContext(GPU{VRAM: 1 * gib}, 40960)).To(Equal(DefaultPhysicalBatch))
+	})
+
+	It("returns the full context (unclamped) when per-device VRAM is unknown", func() {
+		// Unknown VRAM (CPU / detection gap) preserves the original single-pass
+		// behavior — the cap is a downward safety that only engages when VRAM is
+		// known. Clamping here would over-trim score/embed/rerank inputs.
+		Expect(SinglePassBatchForContext(GPU{VRAM: 0}, 40960)).To(Equal(40960))
+	})
+})

core/config/model_capabilities.go

@@ -0,0 +1,197 @@
+package config
+
+// This file is the single source of truth for deriving a model's user-facing
+// capabilities and input/output modalities from its ModelConfig. Both the
+// OpenAI-compatible /v1/models/capabilities endpoint and the Ollama-compatible
+// /api/tags|/api/show surface consume these, so the vocabulary stays consistent
+// across clients. Keep the detection heuristics here rather than duplicating
+// them per endpoint.
+
+// VisionSupported reports whether the model can accept image inputs.
+//
+// We deliberately avoid HasUsecases(FLAG_VISION): GuessUsecases has no
+// FLAG_VISION branch and reports true for any chat model, so it would paint
+// vision onto text-only models. Instead we look for explicit signals: the
+// declared KnownUsecases bit, a multimodal projector, or a template/backend
+// multimodal marker.
+func (c *ModelConfig) VisionSupported() bool {
+	if c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_VISION) == FLAG_VISION {
+		return true
+	}
+	if c.MMProj != "" {
+		return true
+	}
+	if c.TemplateConfig.Multimodal != "" {
+		return true
+	}
+	if c.MediaMarker != "" {
+		return true
+	}
+	return false
+}
+
+// ToolSupported reports whether the model is wired up for tool / function
+// calling. We look for any of the explicit knobs LocalAI uses to drive
+// function-call extraction (regex match, response regex, grammar triggers, XML
+// format) or the auto-detected tool-format markers the llama.cpp backend
+// populates during model load.
+func (c *ModelConfig) ToolSupported() bool {
+	fc := c.FunctionsConfig
+	if fc.ToolFormatMarkers != nil && fc.ToolFormatMarkers.FormatType != "" {
+		return true
+	}
+	if len(fc.JSONRegexMatch) > 0 || len(fc.ResponseRegex) > 0 {
+		return true
+	}
+	if fc.XMLFormatPreset != "" || fc.XMLFormat != nil {
+		return true
+	}
+	if len(fc.GrammarConfig.GrammarTriggers) > 0 || fc.GrammarConfig.SchemaType != "" {
+		return true
+	}
+	return false
+}
+
+// ThinkingSupported reports whether the model has reasoning / thinking enabled.
+// LocalAI sets DisableReasoning=false (or leaves thinking markers configured)
+// when the backend probe reports that the model supports thinking.
+func (c *ModelConfig) ThinkingSupported() bool {
+	rc := c.ReasoningConfig
+	if rc.DisableReasoning != nil && !*rc.DisableReasoning {
+		return true
+	}
+	if len(rc.ThinkingStartTokens) > 0 || len(rc.TagPairs) > 0 {
+		// Explicit thinking markers imply support unless explicitly disabled.
+		return rc.DisableReasoning == nil || !*rc.DisableReasoning
+	}
+	return false
+}
+
+// AudioInputSupported reports whether a chat/generation model accepts audio as
+// input (e.g. vLLM omni models). The signal is the vLLM per-prompt audio limit;
+// there is no FLAG_* for "chat model that hears audio", which is exactly why a
+// plain usecase list can't express it. Transcription models are handled
+// separately in InputModalities via FLAG_TRANSCRIPT.
+func (c *ModelConfig) AudioInputSupported() bool {
+	return c.LimitMMPerPrompt.LimitAudioPerPrompt > 0
+}
+
+// VideoInputSupported reports whether a chat/generation model accepts video as
+// input. The signal is the vLLM per-prompt video limit. Note this is distinct
+// from FLAG_VIDEO, which denotes video *generation* (diffusers) — an output
+// modality, not an input one.
+func (c *ModelConfig) VideoInputSupported() bool {
+	return c.LimitMMPerPrompt.LimitVideoPerPrompt > 0
+}
+
+// Capabilities returns the ordered list of capability strings the model
+// supports, using the canonical usecase vocabulary (chat, vision, transcript,
+// tts, embeddings, image, video, ...) plus the modifier capabilities "tools"
+// and "thinking". Vision is resolved via VisionSupported (not HasUsecases) to
+// avoid the guess-heuristic false positive.
+func (c *ModelConfig) Capabilities() []string {
+	chat := c.HasUsecases(FLAG_CHAT)
+	completion := c.HasUsecases(FLAG_COMPLETION)
+
+	var caps []string
+	add := func(cond bool, name string) {
+		if cond {
+			caps = append(caps, name)
+		}
+	}
+
+	add(chat, UsecaseChat)
+	add(completion, UsecaseCompletion)
+	add(c.HasUsecases(FLAG_EDIT), UsecaseEdit)
+	add(c.HasUsecases(FLAG_EMBEDDINGS), UsecaseEmbeddings)
+	add(c.HasUsecases(FLAG_RERANK), UsecaseRerank)
+	// Vision is only meaningful as an image-understanding modifier on a chat/
+	// completion model. Gating on (chat||completion) matches the Ollama surface
+	// and avoids a false positive when config defaults hydrate a MediaMarker on
+	// a non-chat model (e.g. a pure ASR/TTS backend).
+	add((chat || completion) && c.VisionSupported(), UsecaseVision)
+	// tools/thinking are modifiers on the chat/completion surface.
+	add((chat || completion) && c.ToolSupported(), "tools")
+	add((chat || completion) && c.ThinkingSupported(), "thinking")
+	add(c.HasUsecases(FLAG_TRANSCRIPT), UsecaseTranscript)
+	add(c.HasUsecases(FLAG_TTS), UsecaseTTS)
+	add(c.HasUsecases(FLAG_SOUND_GENERATION), UsecaseSoundGeneration)
+	add(c.HasUsecases(FLAG_IMAGE), UsecaseImage)
+	add(c.HasUsecases(FLAG_VIDEO), UsecaseVideo)
+	add(c.HasUsecases(FLAG_VAD), UsecaseVAD)
+	add(c.HasUsecases(FLAG_DETECTION), UsecaseDetection)
+	add(c.HasUsecases(FLAG_DEPTH), UsecaseDepth)
+	add(c.HasUsecases(FLAG_AUDIO_TRANSFORM), UsecaseAudioTransform)
+	add(c.HasUsecases(FLAG_DIARIZATION), UsecaseDiarization)
+	add(c.HasUsecases(FLAG_SOUND_CLASSIFICATION), UsecaseSoundClassification)
+	add(c.HasUsecases(FLAG_REALTIME_AUDIO), UsecaseRealtimeAudio)
+	add(c.HasUsecases(FLAG_FACE_RECOGNITION), UsecaseFaceRecognition)
+	add(c.HasUsecases(FLAG_SPEAKER_RECOGNITION), UsecaseSpeakerRecognition)
+	return caps
+}
+
+// InputModalities returns the set of modalities (text, image, audio, video) the
+// model accepts as input, ordered text→image→audio→video. This is what an
+// attachment router consults to decide whether an image/audio/video file can be
+// handed to the active model directly.
+func (c *ModelConfig) InputModalities() []string {
+	imageGen := c.HasUsecases(FLAG_IMAGE)
+	videoGen := c.HasUsecases(FLAG_VIDEO)
+	chatish := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION)
+
+	textIn := chatish || c.HasUsecases(FLAG_EDIT) ||
+		c.HasUsecases(FLAG_EMBEDDINGS) || c.HasUsecases(FLAG_RERANK) || c.HasUsecases(FLAG_TOKENIZE) ||
+		c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) || imageGen || videoGen
+
+	// Image input via a chat model requires vision (gated on chat, like the
+	// Ollama surface); detection/depth/face models consume images directly.
+	imageIn := (chatish && c.VisionSupported()) || c.LimitMMPerPrompt.LimitImagePerPrompt > 0 ||
+		c.HasUsecases(FLAG_DETECTION) || c.HasUsecases(FLAG_DEPTH) || c.HasUsecases(FLAG_FACE_RECOGNITION)
+
+	audioIn := c.AudioInputSupported() || c.HasUsecases(FLAG_TRANSCRIPT) || c.HasUsecases(FLAG_AUDIO_TRANSFORM) ||
+		c.HasUsecases(FLAG_REALTIME_AUDIO) || c.HasUsecases(FLAG_VAD) || c.HasUsecases(FLAG_DIARIZATION) ||
+		c.HasUsecases(FLAG_SOUND_CLASSIFICATION) || c.HasUsecases(FLAG_SPEAKER_RECOGNITION)
+
+	videoIn := c.VideoInputSupported()
+
+	var mods []string
+	if textIn {
+		mods = append(mods, "text")
+	}
+	if imageIn {
+		mods = append(mods, "image")
+	}
+	if audioIn {
+		mods = append(mods, "audio")
+	}
+	if videoIn {
+		mods = append(mods, "video")
+	}
+	return mods
+}
+
+// OutputModalities returns the set of modalities (text, image, audio, video)
+// the model produces, ordered text→image→audio→video.
+func (c *ModelConfig) OutputModalities() []string {
+	textOut := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) || c.HasUsecases(FLAG_EDIT) ||
+		c.HasUsecases(FLAG_TRANSCRIPT)
+	imageOut := c.HasUsecases(FLAG_IMAGE)
+	audioOut := c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) ||
+		c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO)
+	videoOut := c.HasUsecases(FLAG_VIDEO)
+
+	var mods []string
+	if textOut {
+		mods = append(mods, "text")
+	}
+	if imageOut {
+		mods = append(mods, "image")
+	}
+	if audioOut {
+		mods = append(mods, "audio")
+	}
+	if videoOut {
+		mods = append(mods, "video")
+	}
+	return mods
+}

core/config/model_capabilities_test.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func usecaseBits(flags ModelConfigUsecase) *ModelConfigUsecase {
+	return &flags
+}
+
+var _ = Describe("Model capabilities derivation", func() {
+	Describe("VisionSupported", func() {
+		It("is false for a plain text chat model", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
+			Expect(cfg.VisionSupported()).To(BeFalse())
+		})
+
+		It("is true when the FLAG_VISION bit is declared", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT | FLAG_VISION), Backend: "llama.cpp"}
+			Expect(cfg.VisionSupported()).To(BeTrue())
+		})
+
+		It("is true when an mmproj projector is set", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
+			cfg.MMProj = "mmproj.gguf" // promoted field from the embedded options struct
+			Expect(cfg.VisionSupported()).To(BeTrue())
+		})
+
+		It("does not fall for the GuessUsecases FLAG_VISION false positive", func() {
+			// A chat model with a chat template would make HasUsecases(FLAG_VISION)
+			// return true via the guess heuristic; VisionSupported must not.
+			cfg := &ModelConfig{Backend: "llama.cpp"}
+			cfg.TemplateConfig.Chat = "{{.Input}}"
+			Expect(cfg.VisionSupported()).To(BeFalse())
+		})
+	})
+
+	Describe("AudioInputSupported / VideoInputSupported", func() {
+		It("detects vLLM omni audio input via limit_mm_per_prompt", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
+			cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1
+			Expect(cfg.AudioInputSupported()).To(BeTrue())
+			Expect(cfg.VideoInputSupported()).To(BeFalse())
+		})
+
+		It("detects vLLM omni video input via limit_mm_per_prompt", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
+			cfg.LimitMMPerPrompt.LimitVideoPerPrompt = 2
+			Expect(cfg.VideoInputSupported()).To(BeTrue())
+		})
+	})
+
+	Describe("Capabilities + modalities", func() {
+		It("a text-only chat model exposes chat and text-only modalities", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
+			Expect(cfg.Capabilities()).To(ContainElement(UsecaseChat))
+			Expect(cfg.Capabilities()).NotTo(ContainElement(UsecaseVision))
+			Expect(cfg.Capabilities()).NotTo(ContainElement(UsecaseTranscript))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"text"}))
+		})
+
+		It("a vision chat model accepts text+image input", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT | FLAG_VISION), Backend: "llama.cpp"}
+			Expect(cfg.Capabilities()).To(ContainElements(UsecaseChat, UsecaseVision))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text", "image"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"text"}))
+		})
+
+		It("an omni chat model accepts text+audio input without an audio capability flag", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
+			cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1
+			// audio-in is a modality, not a usecase string — this is exactly the
+			// case a plain capability list cannot express.
+			Expect(cfg.Capabilities()).To(ContainElement(UsecaseChat))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text", "audio"}))
+		})
+
+		It("a transcription model reads audio and writes text", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TRANSCRIPT), Backend: "parakeet-cpp"}
+			Expect(cfg.Capabilities()).To(Equal([]string{UsecaseTranscript}))
+			Expect(cfg.InputModalities()).To(Equal([]string{"audio"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"text"}))
+		})
+
+		It("an image-generation model reads text and writes an image", func() {
+			// stablediffusion-ggml is image-only; plain "stablediffusion" is also
+			// in GuessUsecases' video-backend list, so it would report video too.
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_IMAGE), Backend: "stablediffusion-ggml"}
+			Expect(cfg.Capabilities()).To(Equal([]string{UsecaseImage}))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"image"}))
+		})
+
+		It("a TTS model reads text and writes audio", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "piper"}
+			Expect(cfg.Capabilities()).To(ContainElement(UsecaseTTS))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"audio"}))
+		})
+	})
+})

core/http/auth/oauth.go

@@ -202,6 +202,11 @@ func (m *OAuthManager) CallbackHandler(providerName string, db *gorm.DB, adminEm
 			userInfo, err = fetchGitHubUserInfoAsOAuth(ctx, token.AccessToken)
 		}
 		if err != nil {
+			// Surface the real cause server-side: ID-token verify failures (issuer/
+			// audience mismatch behind a reverse proxy), a missing id_token, claim
+			// parse errors, or the GitHub userinfo HTTP status/body. The client still
+			// gets the generic message below; details go to logs only. See #10677.
+			xlog.Error("OAuth callback: failed to resolve user info", "provider", providerName, "error", err)
 			return c.JSON(http.StatusInternalServerError, map[string]string{"error": "failed to fetch user info"})
 		}
 

core/http/endpoints/ollama/capabilities.go

@@ -49,62 +49,23 @@ func modelCapabilities(cfg *config.ModelConfig) []string {
 	return caps
 }
 
-// hasVisionSupport reports whether the model can accept image inputs. We avoid
-// cfg.HasUsecases(FLAG_VISION) because GuessUsecases has no FLAG_VISION case
-// and returns true for any chat model — see core/config/model_config.go. Instead
-// we look for explicit signals: KnownUsecases bit, multimodal projector, or
-// template/backend-reported multimodal markers.
+// hasVisionSupport reports whether the model can accept image inputs.
+// The detection heuristic is the canonical config.ModelConfig.VisionSupported —
+// kept as a thin wrapper here so the Ollama capability mapping reads cleanly.
 func hasVisionSupport(cfg *config.ModelConfig) bool {
-	if cfg.KnownUsecases != nil && (*cfg.KnownUsecases&config.FLAG_VISION) == config.FLAG_VISION {
-		return true
-	}
-	if cfg.MMProj != "" {
-		return true
-	}
-	if cfg.TemplateConfig.Multimodal != "" {
-		return true
-	}
-	if cfg.MediaMarker != "" {
-		return true
-	}
-	return false
+	return cfg.VisionSupported()
 }
 
-// hasToolSupport reports whether the model is wired up for tool / function calling.
-// We look for any of the explicit configuration knobs LocalAI uses to drive
-// function-call extraction (regex match, response regex, grammar triggers, XML
-// format) or for the auto-detected tool-format markers populated by the
-// llama.cpp backend during model load.
+// hasToolSupport reports whether the model is wired up for tool / function
+// calling. Delegates to the canonical config.ModelConfig.ToolSupported.
 func hasToolSupport(cfg *config.ModelConfig) bool {
-	fc := cfg.FunctionsConfig
-	if fc.ToolFormatMarkers != nil && fc.ToolFormatMarkers.FormatType != "" {
-		return true
-	}
-	if len(fc.JSONRegexMatch) > 0 || len(fc.ResponseRegex) > 0 {
-		return true
-	}
-	if fc.XMLFormatPreset != "" || fc.XMLFormat != nil {
-		return true
-	}
-	if len(fc.GrammarConfig.GrammarTriggers) > 0 || fc.GrammarConfig.SchemaType != "" {
-		return true
-	}
-	return false
+	return cfg.ToolSupported()
 }
 
 // hasThinkingSupport reports whether the model has reasoning / thinking enabled.
-// LocalAI sets DisableReasoning=false (or leaves thinking markers configured)
-// when the backend probe reports that the model supports thinking.
+// Delegates to the canonical config.ModelConfig.ThinkingSupported.
 func hasThinkingSupport(cfg *config.ModelConfig) bool {
-	rc := cfg.ReasoningConfig
-	if rc.DisableReasoning != nil && !*rc.DisableReasoning {
-		return true
-	}
-	if len(rc.ThinkingStartTokens) > 0 || len(rc.TagPairs) > 0 {
-		// Explicit thinking markers imply support unless explicitly disabled.
-		return rc.DisableReasoning == nil || !*rc.DisableReasoning
-	}
-	return false
+	return cfg.ThinkingSupported()
 }
 
 // quantRegex matches GGUF-style quantization suffixes (Q4_K_M, Q8_0, IQ3_XS, F16, ...).

core/http/endpoints/openai/list.go

@@ -21,48 +21,11 @@ func ListModelsEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader, ap
 		authDB = db[0]
 	}
 	return func(c echo.Context) error {
-		// If blank, no filter is applied.
-		filter := c.QueryParam("filter")
-
-		// By default, exclude any loose files that are already referenced by a configuration file.
-		var policy galleryop.LooseFilePolicy
-		excludeConfigured := c.QueryParam("excludeConfigured")
-		if excludeConfigured == "" || excludeConfigured == "true" {
-			policy = galleryop.SKIP_IF_CONFIGURED
-		} else {
-			policy = galleryop.ALWAYS_INCLUDE // This replicates current behavior. TODO: give more options to the user?
-		}
-
-		filterFn, err := config.BuildNameFilterFn(filter)
+		modelNames, err := listVisibleModelNames(c, bcl, ml, authDB)
 		if err != nil {
 			return err
 		}
 
-		modelNames, err := galleryop.ListModels(bcl, ml, filterFn, policy)
-		if err != nil {
-			return err
-		}
-
-		// Filter models by user's allowlist if auth is enabled
-		if authDB != nil {
-			if user := auth.GetUser(c); user != nil && user.Role != auth.RoleAdmin {
-				perm, err := auth.GetCachedUserPermissions(c, authDB, user.ID)
-				if err == nil && perm.AllowedModels.Enabled {
-					allowed := map[string]bool{}
-					for _, m := range perm.AllowedModels.Models {
-						allowed[m] = true
-					}
-					filtered := make([]string, 0, len(modelNames))
-					for _, m := range modelNames {
-						if allowed[m] {
-							filtered = append(filtered, m)
-						}
-					}
-					modelNames = filtered
-				}
-			}
-		}
-
 		// Map from a slice of names to a slice of OpenAIModel response objects
 		dataModels := []schema.OpenAIModel{}
 		for _, m := range modelNames {
@@ -75,3 +38,53 @@ func ListModelsEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader, ap
 		})
 	}
 }
+
+// listVisibleModelNames resolves the model names visible to the caller, applying
+// the same query filters (filter, excludeConfigured) and per-user allowlist as
+// the OpenAI models listing. Shared by ListModelsEndpoint and
+// ListModelCapabilitiesEndpoint so both stay consistent.
+func listVisibleModelNames(c echo.Context, bcl *config.ModelConfigLoader, ml *model.ModelLoader, authDB *gorm.DB) ([]string, error) {
+	// If blank, no filter is applied.
+	filter := c.QueryParam("filter")
+
+	// By default, exclude any loose files that are already referenced by a configuration file.
+	var policy galleryop.LooseFilePolicy
+	excludeConfigured := c.QueryParam("excludeConfigured")
+	if excludeConfigured == "" || excludeConfigured == "true" {
+		policy = galleryop.SKIP_IF_CONFIGURED
+	} else {
+		policy = galleryop.ALWAYS_INCLUDE // This replicates current behavior. TODO: give more options to the user?
+	}
+
+	filterFn, err := config.BuildNameFilterFn(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	modelNames, err := galleryop.ListModels(bcl, ml, filterFn, policy)
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter models by user's allowlist if auth is enabled
+	if authDB != nil {
+		if user := auth.GetUser(c); user != nil && user.Role != auth.RoleAdmin {
+			perm, err := auth.GetCachedUserPermissions(c, authDB, user.ID)
+			if err == nil && perm.AllowedModels.Enabled {
+				allowed := map[string]bool{}
+				for _, m := range perm.AllowedModels.Models {
+					allowed[m] = true
+				}
+				filtered := make([]string, 0, len(modelNames))
+				for _, m := range modelNames {
+					if allowed[m] {
+						filtered = append(filtered, m)
+					}
+				}
+				modelNames = filtered
+			}
+		}
+	}
+
+	return modelNames, nil
+}

core/http/endpoints/openai/list_capabilities.go

@@ -0,0 +1,50 @@
+package openai
+
+import (
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+	model "github.com/mudler/LocalAI/pkg/model"
+	"gorm.io/gorm"
+)
+
+// ListModelCapabilitiesEndpoint is a LocalAI-specific extension of the OpenAI
+// models listing. It returns the same set of models as /v1/models but enriches
+// each entry with the capabilities and input/output modalities the model
+// supports, so clients can decide whether an image/audio/video attachment can be
+// handed to a given model directly (or must be converted/transcribed first).
+//
+// It is purely additive: clients that don't know about it keep using /v1/models
+// and see no change.
+// @Summary List available models enriched with capabilities and input/output modalities.
+// @Tags models
+// @Success 200 {object} schema.ModelCapabilitiesResponse "Response"
+// @Router /v1/models/capabilities [get]
+func ListModelCapabilitiesEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, db ...*gorm.DB) echo.HandlerFunc {
+	var authDB *gorm.DB
+	if len(db) > 0 {
+		authDB = db[0]
+	}
+	return func(c echo.Context) error {
+		modelNames, err := listVisibleModelNames(c, bcl, ml, authDB)
+		if err != nil {
+			return err
+		}
+
+		dataModels := []schema.ModelCapabilities{}
+		for _, m := range modelNames {
+			entry := schema.ModelCapabilities{ID: m, Object: "model"}
+			if cfg, ok := bcl.GetModelConfig(m); ok {
+				entry.Capabilities = cfg.Capabilities()
+				entry.InputModalities = cfg.InputModalities()
+				entry.OutputModalities = cfg.OutputModalities()
+			}
+			dataModels = append(dataModels, entry)
+		}
+
+		return c.JSON(200, schema.ModelCapabilitiesResponse{
+			Object: "list",
+			Data:   dataModels,
+		})
+	}
+}

core/http/endpoints/openai/list_capabilities_test.go

@@ -0,0 +1,119 @@
+package openai
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ListModelCapabilitiesEndpoint", func() {
+	var (
+		e       *echo.Echo
+		tmpDir  string
+		bcl     *config.ModelConfigLoader
+		ml      *model.ModelLoader
+		appConf *config.ApplicationConfig
+	)
+
+	BeforeEach(func() {
+		var err error
+		e = echo.New()
+		tmpDir, err = os.MkdirTemp("", "models-caps-test-*")
+		Expect(err).NotTo(HaveOccurred())
+
+		st, err := system.GetSystemState(system.WithModelPath(tmpDir))
+		Expect(err).NotTo(HaveOccurred())
+		ml = model.NewModelLoader(st)
+		bcl = config.NewModelConfigLoader(tmpDir)
+		appConf = config.NewApplicationConfig()
+	})
+
+	AfterEach(func() {
+		_ = os.RemoveAll(tmpDir)
+	})
+
+	writeConfig := func(name, yaml string) {
+		path := filepath.Join(tmpDir, name+".yaml")
+		Expect(os.WriteFile(path, []byte(yaml), 0o644)).To(Succeed())
+		Expect(bcl.ReadModelConfig(path)).To(Succeed())
+	}
+
+	// call exercises the endpoint with auth disabled (no auth DB), which is the
+	// standard deployment path. The per-user allowlist branch is shared verbatim
+	// with ListModelsEndpoint (listVisibleModelNames) and covered there.
+	call := func() schema.ModelCapabilitiesResponse {
+		req := httptest.NewRequest(http.MethodGet, "/v1/models/capabilities", nil)
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+
+		handler := ListModelCapabilitiesEndpoint(bcl, ml, appConf)
+		Expect(handler(c)).To(Succeed())
+		Expect(rec.Code).To(Equal(http.StatusOK))
+
+		var resp schema.ModelCapabilitiesResponse
+		Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed())
+		return resp
+	}
+
+	entryFor := func(resp schema.ModelCapabilitiesResponse, id string) *schema.ModelCapabilities {
+		for i := range resp.Data {
+			if resp.Data[i].ID == id {
+				return &resp.Data[i]
+			}
+		}
+		return nil
+	}
+
+	It("returns the list envelope even with no models", func() {
+		resp := call()
+		Expect(resp.Object).To(Equal("list"))
+	})
+
+	It("enriches a vision chat model with capabilities and image input modality", func() {
+		writeConfig("vlm", `
+name: vlm
+backend: llama-cpp
+known_usecases:
+  - FLAG_CHAT
+  - FLAG_VISION
+template:
+  chat: "{{ .Input }}"
+parameters:
+  model: qwen2.5-vl-Q4_K_M.gguf
+`)
+		entry := entryFor(call(), "vlm")
+		Expect(entry).NotTo(BeNil())
+		Expect(entry.Object).To(Equal("model"))
+		Expect(entry.Capabilities).To(ContainElements("chat", "vision"))
+		Expect(entry.InputModalities).To(ContainElements("text", "image"))
+		Expect(entry.OutputModalities).To(ContainElement("text"))
+	})
+
+	It("marks a parakeet model as an audio-in/text-out transcription model", func() {
+		writeConfig("parakeet", `
+name: parakeet
+backend: parakeet-cpp
+known_usecases:
+  - FLAG_TRANSCRIPT
+parameters:
+  model: parakeet-tdt-0.6b
+`)
+		entry := entryFor(call(), "parakeet")
+		Expect(entry).NotTo(BeNil())
+		Expect(entry.Capabilities).To(ContainElement("transcript"))
+		Expect(entry.InputModalities).To(Equal([]string{"audio"}))
+		Expect(entry.OutputModalities).To(Equal([]string{"text"}))
+		Expect(entry.Capabilities).NotTo(ContainElement("chat"))
+	})
+})

core/http/react-ui/src/App.css

@@ -6304,6 +6304,9 @@ select.input {
 .home-wizard {
   max-width: 48rem;
   width: 100%;
+  /* .home-page is a stretch column flex; a max-width child would otherwise
+     pin to the left cross-start edge. Center it. */
+  margin: 0 auto;
 }
 .home-wizard-hero {
   text-align: center;

core/http/routes/localai.go

@@ -272,25 +272,27 @@ func RegisterLocalAIRoutes(router *echo.Echo,
 			"version": internal.PrintableVersion(),
 			// Flat endpoint list for backwards compatibility
 			"endpoints": map[string]any{
-				"models":           "/v1/models",
-				"chat_completions": "/v1/chat/completions",
-				"completions":      "/v1/completions",
-				"embeddings":       "/v1/embeddings",
-				"config_metadata":  "/api/models/config-metadata",
-				"config_json":      "/api/models/config-json/:name",
-				"config_patch":     "/api/models/config-json/:name",
-				"autocomplete":     "/api/models/config-metadata/autocomplete/:provider",
-				"vram_estimate":    "/api/models/vram-estimate",
-				"tts":              "/tts",
-				"transcription":    "/v1/audio/transcriptions",
-				"image_generation": "/v1/images/generations",
-				"swagger":          "/swagger/index.html",
-				"instructions":     "/api/instructions",
+				"models":              "/v1/models",
+				"models_capabilities": "/v1/models/capabilities",
+				"chat_completions":    "/v1/chat/completions",
+				"completions":         "/v1/completions",
+				"embeddings":          "/v1/embeddings",
+				"config_metadata":     "/api/models/config-metadata",
+				"config_json":         "/api/models/config-json/:name",
+				"config_patch":        "/api/models/config-json/:name",
+				"autocomplete":        "/api/models/config-metadata/autocomplete/:provider",
+				"vram_estimate":       "/api/models/vram-estimate",
+				"tts":                 "/tts",
+				"transcription":       "/v1/audio/transcriptions",
+				"image_generation":    "/v1/images/generations",
+				"swagger":             "/swagger/index.html",
+				"instructions":        "/api/instructions",
 			},
 			// Categorized endpoint groups for structured discovery
 			"endpoint_groups": map[string]any{
 				"openai_compatible": map[string]string{
 					"models":               "/v1/models",
+					"models_capabilities":  "/v1/models/capabilities",
 					"chat_completions":     "/v1/chat/completions",
 					"completions":          "/v1/completions",
 					"embeddings":           "/v1/embeddings",

core/http/routes/openai.go

@@ -257,4 +257,10 @@ func RegisterOpenAIRoutes(app *echo.Echo,
 	// List models
 	app.GET("/v1/models", openai.ListModelsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.AuthDB()))
 	app.GET("/models", openai.ListModelsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.AuthDB()))
+
+	// List models enriched with capabilities + input/output modalities
+	// (LocalAI-specific, additive superset of /v1/models).
+	capabilitiesHandler := openai.ListModelCapabilitiesEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.AuthDB())
+	app.GET("/v1/models/capabilities", capabilitiesHandler)
+	app.GET("/models/capabilities", capabilitiesHandler)
 }

core/schema/openai.go

@@ -251,3 +251,27 @@ type ModelsDataResponse struct {
 	Object string        `json:"object"`
 	Data   []OpenAIModel `json:"data"`
 }
+
+// ModelCapabilities is a strict superset of OpenAIModel that additionally
+// describes what a model can do and which modalities it accepts/produces. It is
+// served by the LocalAI-specific /v1/models/capabilities endpoint so clients can
+// route attachments (image/audio/video) to a model only when it can handle them.
+type ModelCapabilities struct {
+	ID     string `json:"id"`
+	Object string `json:"object"`
+	// Capabilities are canonical usecase strings (e.g. chat, vision, transcript,
+	// tts, embeddings, image, video) plus the modifiers "tools" and "thinking".
+	Capabilities []string `json:"capabilities"`
+	// InputModalities is the subset of {text,image,audio,video} the model accepts.
+	InputModalities []string `json:"input_modalities"`
+	// OutputModalities is the subset of {text,image,audio,video} the model produces.
+	OutputModalities []string `json:"output_modalities"`
+}
+
+// ModelCapabilitiesResponse is the envelope returned by /v1/models/capabilities.
+// It mirrors ModelsDataResponse so a client can treat it as an enriched
+// drop-in for /v1/models.
+type ModelCapabilitiesResponse struct {
+	Object string              `json:"object"`
+	Data   []ModelCapabilities `json:"data"`
+}

core/services/agentpool/agent_pool.go

@@ -426,7 +426,15 @@ func (s *AgentPoolService) Chat(name, message string) (string, error) {
 
 	// Process asynchronously
 	go func() {
+		started := time.Now()
 		response := ag.Ask(coreTypes.WithText(message))
+		outcome := "completed"
+		if response == nil {
+			outcome = "cancelled"
+		} else if response.Error != nil {
+			outcome = "error"
+		}
+		recordAgentRun(name, outcome, time.Since(started).Seconds())
 
 		if response == nil {
 			errMsg, _ := json.Marshal(map[string]any{

core/services/agentpool/metrics.go

@@ -0,0 +1,54 @@
+package agentpool
+
+import (
+	"context"
+	"sync"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// Prometheus metrics for agent chat runs. Operators need a scrape-friendly
+// signal for "are agent turns completing, erroring or getting cancelled,
+// and how long do they take" — log-derived counters proved brittle
+// (ANSI/timezone parsing, container-restart gaps). Chat() is the single
+// choke point of the local execution path, so instrumenting the response
+// handoff covers UI chats, API chats and connector-triggered asks alike.
+//
+// Lazily initialised on first record so the package works no matter when
+// (or whether) the Prometheus-backed global MeterProvider is installed —
+// same pattern as core/services/routing/pii.
+var (
+	agentMetricsOnce sync.Once
+	runsCounter      metric.Int64Counter
+	runSeconds       metric.Float64Histogram
+)
+
+func recordAgentRun(agent, outcome string, seconds float64) {
+	agentMetricsOnce.Do(func() {
+		meter := otel.Meter("github.com/mudler/LocalAI")
+		if c, err := meter.Int64Counter(
+			"localai_agent_runs_total",
+			metric.WithDescription("Agent chat runs, labeled by agent and outcome (completed|error|cancelled)"),
+		); err == nil {
+			runsCounter = c
+		}
+		if h, err := meter.Float64Histogram(
+			"localai_agent_run_seconds",
+			metric.WithDescription("Wall-clock duration of agent chat runs in seconds"),
+		); err == nil {
+			runSeconds = h
+		}
+	})
+	attrs := metric.WithAttributes(
+		attribute.String("agent", agent),
+		attribute.String("outcome", outcome),
+	)
+	if runsCounter != nil {
+		runsCounter.Add(context.Background(), 1, attrs)
+	}
+	if runSeconds != nil {
+		runSeconds.Record(context.Background(), seconds, attrs)
+	}
+}

docs/content/features/api-discovery.md

@@ -36,6 +36,7 @@ Returns the instance version, all available endpoint URLs (flat and categorized)
   "endpoints": {
     "chat_completions": "/v1/chat/completions",
     "models": "/v1/models",
+    "models_capabilities": "/v1/models/capabilities",
     "config_metadata": "/api/models/config-metadata",
     "instructions": "/api/instructions",
     "swagger": "/swagger/index.html"
@@ -123,6 +124,45 @@ Add `?format=json` to get a raw **OpenAPI fragment** (filtered Swagger spec with
 curl http://localhost:8080/api/instructions/config-management?format=json
 ```
 
+## Model Capabilities
+
+`GET /v1/models/capabilities`
+
+An additive, LocalAI-specific superset of `/v1/models`. It returns the same set of models but enriches each entry with the **capabilities** the model supports and the **input/output modalities** it accepts and produces. Use it to decide, before sending a request, whether a given model can take an image, audio, or video attachment directly — or whether the input needs converting/transcribing first.
+
+Because it is purely additive, clients that only understand `/v1/models` keep working unchanged; they simply never call this route.
+
+```bash
+curl http://localhost:8080/v1/models/capabilities
+```
+
+```json
+{
+  "object": "list",
+  "data": [
+    {
+      "id": "qwen2.5-omni",
+      "object": "model",
+      "capabilities": ["chat", "vision", "tools"],
+      "input_modalities": ["text", "image", "audio"],
+      "output_modalities": ["text"]
+    },
+    {
+      "id": "parakeet",
+      "object": "model",
+      "capabilities": ["transcript"],
+      "input_modalities": ["audio"],
+      "output_modalities": ["text"]
+    }
+  ]
+}
+```
+
+- **`capabilities`** — canonical usecase strings (e.g. `chat`, `vision`, `transcript`, `tts`, `embeddings`, `image`, `video`) plus the modifiers `tools` and `thinking`.
+- **`input_modalities` / `output_modalities`** — the subsets of `{text, image, audio, video}` the model accepts and produces. Audio and video *input* are derived from the model's multimodal limits (e.g. vLLM `limit_mm_per_prompt`), which no single usecase flag expresses — which is why this endpoint exists alongside the plain listing.
+
+The same query parameters as `/v1/models` are honored (`filter`, `excludeConfigured`), and the same per-user model allowlist is applied when authentication is enabled.
+
 ## Configuration Management APIs
 
 These endpoints let agents discover model configuration fields, read current settings, modify them, and estimate VRAM usage.

docs/content/reference/cli-reference.md

@@ -23,8 +23,8 @@ Complete reference for all LocalAI command-line interface (CLI) parameters and e
 |-----------|---------|-------------|----------------------|
 | `--models-path` | `BASEPATH/models` | Path containing models used for inferencing | `$LOCALAI_MODELS_PATH`, `$MODELS_PATH` |
 | `--data-path` | `BASEPATH/data` | Path for persistent data (collectiondb, agent state, tasks, jobs). Separates mutable data from configuration | `$LOCALAI_DATA_PATH` |
-| `--generated-content-path` | `/tmp/generated/content` | Location for assets generated by backends (e.g. stablediffusion, images, audio, videos) | `$LOCALAI_GENERATED_CONTENT_PATH`, `$GENERATED_CONTENT_PATH` |
-| `--upload-path` | `/tmp/localai/upload` | Path to store uploads from files API | `$LOCALAI_UPLOAD_PATH`, `$UPLOAD_PATH` |
+| `--generated-content-path` | `TMPDIR/localai-UID/generated/content` | Location for assets generated by backends (e.g. stablediffusion, images, audio, videos). Defaults under the OS temp dir (`$TMPDIR`, falling back to `/tmp`), scoped to the current user's UID so accounts sharing a host never collide. | `$LOCALAI_GENERATED_CONTENT_PATH`, `$GENERATED_CONTENT_PATH` |
+| `--upload-path` | `TMPDIR/localai-UID/upload` | Path to store uploads from files API. Defaults under the OS temp dir (`$TMPDIR`, falling back to `/tmp`), scoped to the current user's UID. | `$LOCALAI_UPLOAD_PATH`, `$UPLOAD_PATH` |
 | `--localai-config-dir` | `BASEPATH/configuration` | Directory for dynamic loading of certain configuration files (currently runtime_settings.json, api_keys.json, and external_backends.json). See [Runtime Settings]({{%relref "features/runtime-settings" %}}) for web-based configuration. | `$LOCALAI_CONFIG_DIR` |
 | `--localai-config-dir-poll-interval` | | Time duration to poll the LocalAI Config Dir if your system has broken fsnotify events (example: `1m`) | `$LOCALAI_CONFIG_DIR_POLL_INTERVAL` |
 | `--models-config-file` | | YAML file containing a list of model backend configs (alias: `--config-file`) | `$LOCALAI_MODELS_CONFIG_FILE`, `$CONFIG_FILE` |

docs/content/whats-new.md

@@ -17,6 +17,7 @@ You can see the release notes [here](https://github.com/mudler/LocalAI/releases)
 - **May 2026**: [Speaker diarization](/features/audio-diarization/) — new `/v1/audio/diarization` endpoint returning "who spoke when" segments. Backed by `sherpa-onnx` (pyannote-3.0 + speaker embeddings + clustering) for pure diarization, and `vibevoice-cpp` for diarization bundled with long-form ASR. Supports `json` / `verbose_json` / `rttm` response formats.
 - **June 2026**: [Sound classification](/features/audio-classification/) — new `/v1/audio/classification` endpoint for audio tagging / sound-event classification, returning scored [AudioSet](https://research.google.com/audioset/) labels (baby cry, glass breaking, alarms, ...). Backed by [ced.cpp](https://github.com/mudler/ced.cpp), a 527-class AudioSet tagger ported to ggml.
 - **June 2026**: [PII analyze / redact API](/features/middleware/#analyze--redact-api) — the PII detection pipeline (NER + restricted-regex pattern tiers) is now a standalone service: `POST /api/pii/analyze` returns detected entity spans and `POST /api/pii/redact` returns the sanitised text (or `400 pii_blocked`), without routing a chat request through the middleware. Events gain an `origin` (`middleware` / `proxy` / `pii_analyze` / `pii_redact`) so `/api/pii/events` can be filtered by source.
+- **July 2026**: [Model capabilities endpoint](/features/api-discovery/#model-capabilities) — `GET /v1/models/capabilities`, an additive superset of `/v1/models` that reports each model's `capabilities` plus its `input_modalities` / `output_modalities` (`text` / `image` / `audio` / `video`). Lets clients route image/audio/video attachments to a model only when it can handle them; audio/video *input* is derived from the model's multimodal limits, which no single usecase flag expresses.
 - **June 2026**: Concurrent scoring and PII NER on llama.cpp — the `Score` (router classifier) and `TokenClassify` (PII NER) primitives now ride llama.cpp's server task queue instead of locking the context, so they run concurrently with chat/completion/embedding traffic and with each other. The `known_usecases` restriction that forced dedicated scorer/NER model configs on llama-cpp is lifted, repeated scoring calls reuse the prompt KV cache across candidates, and scoring inputs are no longer capped by the physical batch size.
 
 ## 2024 Highlights

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v4.5.6"
+  "version": "v4.6.0"
 }

gallery/index.yaml

@@ -3,22 +3,7 @@
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
     - https://huggingface.co/Jackrong/Qwopus3.6-35B-A3B-Coder-MTP-GGUF
-  description: |
-    # 🌟 Qwopus3.6-35B-A3B-v1
-
-    ## 💡 Base Model Overview
-
-    **Qwen3.6-35B-A3B** is an advanced hybrid sparse MoE (Mixture-of-Experts) model developed by Alibaba Cloud. It features 35B total parameters with only 3B active parameters per token, ensuring high inference efficiency. Architecturally, it combines Gated DeltaNet linear attention with standard gated attention layers, routing tokens across **256 experts**. It natively supports a massive **262k context window** and is specifically designed for high-performance agentic coding, deep reasoning, and multimodal tasks.
-
-    ## 🚀 Model Refinement & Logic Tuning （Qwopus3.6-35B-A3B-v1）
-
-    🪐**Qwopus3.6-35B-A3B-v1** is a reasoning-enhanced MoE (Mixture of Experts) model fine-tuned on top of **Qwen3.6-35B-A3B**.
-
-    ### 🛠 Training Strategy
-
-    The fine-tuning process for this model is structured into **three distinct stages of distributed SFT (Supervised Fine-Tuning)**, progressively scaling reasoning complexity and data diversity. This systematic approach ensures the model inherits the base MoE capabilities while sharpening its logic-handling depth.
-
-    ...
+  description: "# \U0001F31F Qwopus3.6-35B-A3B-v1\n\n## \U0001F4A1 Base Model Overview\n\n**Qwen3.6-35B-A3B** is an advanced hybrid sparse MoE (Mixture-of-Experts) model developed by Alibaba Cloud. It features 35B total parameters with only 3B active parameters per token, ensuring high inference efficiency. Architecturally, it combines Gated DeltaNet linear attention with standard gated attention layers, routing tokens across **256 experts**. It natively supports a massive **262k context window** and is specifically designed for high-performance agentic coding, deep reasoning, and multimodal tasks.\n\n## \U0001F680 Model Refinement & Logic Tuning （Qwopus3.6-35B-A3B-v1）\n\n\U0001FA90**Qwopus3.6-35B-A3B-v1** is a reasoning-enhanced MoE (Mixture of Experts) model fine-tuned on top of **Qwen3.6-35B-A3B**.\n\n### \U0001F6E0 Training Strategy\n\nThe fine-tuning process for this model is structured into **three distinct stages of distributed SFT (Supervised Fine-Tuning)**, progressively scaling reasoning complexity and data diversity. This systematic approach ensures the model inherits the base MoE capabilities while sharpening its logic-handling depth.\n\n...\n"
   license: "apache-2.0"
   tags:
     - llm
@@ -55,34 +40,7 @@
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
     - https://huggingface.co/protoLabsAI/Ornith-1.0-9B-MTP-GGUF
-  description: |
-    [](https://deep-reinforce.com/ornith.html)
-
-    # Ornith-1.0-9B
-
-    Aloha! 🌺 Today, we are releasing Ornith-1.0, a self-improving family of open-source models for agentic coding.
-
-    Highlights:
-
-      - **State-of-the-Art Coding Agents**: Available in 9B-Dense, 31B-Dense, 35B-MoE, and 397B-MoE (post-trained on top of Gemma 4 and Qwen 3.5), achieving state-of-the-art performance among open-source models of comparable size on coding benchmarks such as Terminal-Bench 2.1, SWE-Bench, NL2Repo and OpenClaw.
-      - **Self-Improving Training Framework**:  Ornith-1.0 employs RL to learn to generate not only solution rollouts, but also the scallfold that drive those rollouts. By jointly optimizing the scaffold and the resulting solution, the model  discovers better search trajectories and generates higher-quality solutions.
-      - **Licence**: MIT licensed, globally accessible, and free from regional limitations.
-
-    ## Ornith 1.0 9B
-
-    This model card documents **Ornith-1.0-9B**, the most lightweight member of the Ornith family, designed for efficient single-GPU deployment.
-
-    ### Benchmarks
-
-    Ornith-1.0-9B
-    Qwen3.5-9B
-    Qwen3.5-35B
-    Gemma4-12B
-    Gemma4-31B
-
-    Agentic Coding
-
-    ...
+  description: "[](https://deep-reinforce.com/ornith.html)\n\n# Ornith-1.0-9B\n\nAloha! \U0001F33A Today, we are releasing Ornith-1.0, a self-improving family of open-source models for agentic coding.\n\nHighlights:\n\n  - **State-of-the-Art Coding Agents**: Available in 9B-Dense, 31B-Dense, 35B-MoE, and 397B-MoE (post-trained on top of Gemma 4 and Qwen 3.5), achieving state-of-the-art performance among open-source models of comparable size on coding benchmarks such as Terminal-Bench 2.1, SWE-Bench, NL2Repo and OpenClaw.\n  - **Self-Improving Training Framework**:  Ornith-1.0 employs RL to learn to generate not only solution rollouts, but also the scallfold that drive those rollouts. By jointly optimizing the scaffold and the resulting solution, the model  discovers better search trajectories and generates higher-quality solutions.\n  - **Licence**: MIT licensed, globally accessible, and free from regional limitations.\n\n## Ornith 1.0 9B\n\nThis model card documents **Ornith-1.0-9B**, the most lightweight member of the Ornith family, designed for efficient single-GPU deployment.\n\n### Benchmarks\n\nOrnith-1.0-9B\nQwen3.5-9B\nQwen3.5-35B\nGemma4-12B\nGemma4-31B\n\nAgentic Coding\n\n...\n"
   license: "mit"
   tags:
     - llm
@@ -36066,7 +36024,7 @@
   files:
     - filename: parakeet-tdt-0.6b-ja.gguf
       uri: huggingface://cstr/parakeet-tdt-0.6b-ja-GGUF/parakeet-tdt-0.6b-ja.gguf
-      sha256: a9c43116b180b8a2ada2771ac829cf751b9e73adcbe69b7c8379593f9e5da31e
+      sha256: 374eb0132eebaec4df77a9631cbbeb03790be48a4a517f6cc8e8bdb38fe9a584
 - name: parakeet-tdt-1.1b-crispasr
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:
@@ -36525,7 +36483,7 @@
   files:
     - filename: vibevoice-realtime-0.5b-q4_k.gguf
       uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf
-      sha256: e3244986d8939a9a8f65701196efbfe3f8b81afd307b29f434fe259b9c411ef1
+      sha256: 483e1922a9077e3fc66b7947a4d6fee3dfd8edc30afde3410efa5bb386bc0392
 - name: chatterbox-tts-crispasr
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:

pkg/xsysinfo/clinfo.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os/exec"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -49,12 +50,7 @@ type clinfoTypeProp struct {
 }
 
 func (t clinfoTypeProp) isGPU() bool {
-	for _, s := range t.Type {
-		if s == clDeviceTypeGPU {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(t.Type, clDeviceTypeGPU)
 }
 
 // clinfoOnce caches the result for the process lifetime. GPU hardware

scripts/build/package-gpu-libs-rocm-data_test.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Regression test for scripts/build/package-gpu-libs.sh ROCm data bundling.
+#
+# Guards issue #10660: hipBLASLt (rocblaslt) resolves its TensileLibrary_lazy_gfx*.dat
+# kernel data relative to the bundled libhipblaslt.so. The packager copied the
+# rocblas/ data dir but not the hipblaslt/ data dir, so the bundled backend
+# fell back to slow generic kernels and logged
+#   rocblaslt error: Cannot read "TensileLibrary_lazy_gfx1201.dat": No such file or directory
+#
+# This test fabricates a fake ROCm tree containing both rocblas/ and hipblaslt/
+# tensile data, points the packager at it via ROCM_BASE_DIRS, and asserts BOTH
+# data directories are bundled into the target lib dir.
+set -euo pipefail
+
+CURDIR=$(dirname "$(realpath "$0")")
+SCRIPT="$CURDIR/package-gpu-libs.sh"
+
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Fabricate a fake ROCm install with both rocblas and hipblaslt tensile data.
+FAKE_ROCM="$WORK/opt/rocm"
+mkdir -p "$FAKE_ROCM/lib/rocblas/library"
+mkdir -p "$FAKE_ROCM/lib/hipblaslt/library"
+echo "fake rocblas tensile" > "$FAKE_ROCM/lib/rocblas/library/TensileLibrary_lazy_gfx1201.dat"
+echo "fake hipblaslt tensile" > "$FAKE_ROCM/lib/hipblaslt/library/TensileLibrary_lazy_gfx1201.dat"
+
+TARGET="$WORK/target"
+mkdir -p "$TARGET"
+
+# shellcheck source=/dev/null
+source "$SCRIPT" "$TARGET"
+
+# Point the data-dir copy at the fabricated tree instead of the real /opt/rocm,
+# then run the actual ROCm packager. This asserts package_rocm_libs itself
+# bundles BOTH data dirs, not just that the helper works in isolation.
+export BUILD_TYPE=hipblas
+export ROCM_BASE_DIRS="$FAKE_ROCM"
+package_rocm_libs
+
+fail=false
+if [ ! -e "$TARGET/rocblas/library/TensileLibrary_lazy_gfx1201.dat" ]; then
+    echo "FAIL: rocblas tensile data was NOT bundled"
+    fail=true
+fi
+if [ ! -e "$TARGET/hipblaslt/library/TensileLibrary_lazy_gfx1201.dat" ]; then
+    echo "FAIL: hipblaslt tensile data was NOT bundled (regression of #10660)"
+    fail=true
+fi
+
+if [ "$fail" = true ]; then
+    ls -R "$TARGET" || true
+    exit 1
+fi
+
+echo "PASS: rocblas and hipblaslt tensile data were both bundled"
+exit 0

scripts/build/package-gpu-libs.sh

@@ -224,6 +224,50 @@ package_cuda_libs() {
     echo "CUDA libraries packaged successfully"
 }
 
+# Copy a ROCm library data subdirectory (e.g. rocblas, hipblaslt) into the
+# bundled lib/ dir. These directories hold the TensileLibrary_*.dat GPU kernel
+# tuning files, which rocBLAS/hipBLASLt load at runtime *relative to their own
+# .so*. Since backends ship their own copies of libhipblaslt.so/librocblas.so
+# under lib/, the matching data dir must travel with them or the libs fall back
+# to slow generic kernels (rocblaslt error: Cannot read TensileLibrary_lazy_gfx*.dat;
+# see issue #10660).
+#
+# The ROCm search roots default to /opt/rocm{,-*} but can be overridden via the
+# ROCM_BASE_DIRS env var (space-separated), which keeps the copy unit-testable
+# without a real ROCm install.
+# Args: $1 = data subdir name found under <rocm-root>/lib{,64}/
+copy_rocm_data_dir() {
+    local data_name="$1"
+    # Single-line `local x=$(...)` on purpose: `local` masks the command
+    # substitution's exit status, which is 1 when nullglob is unset and would
+    # otherwise trip the script's `set -e`.
+    local old_nullglob=$(shopt -p nullglob)
+    shopt -s nullglob
+    local rocm_dirs
+    if [ -n "${ROCM_BASE_DIRS:-}" ]; then
+        # shellcheck disable=SC2206  # intentional word-split of the override
+        rocm_dirs=(${ROCM_BASE_DIRS})
+    else
+        rocm_dirs=(/opt/rocm /opt/rocm-*)
+    fi
+    eval "$old_nullglob"
+    local found=false
+    local rocm_base lib_subdir
+    for rocm_base in "${rocm_dirs[@]}"; do
+        for lib_subdir in lib lib64; do
+            if [ -d "$rocm_base/$lib_subdir/$data_name" ]; then
+                echo "Found $data_name data at $rocm_base/$lib_subdir/$data_name"
+                mkdir -p "$TARGET_LIB_DIR/$data_name"
+                cp -arfL "$rocm_base/$lib_subdir/$data_name/"* "$TARGET_LIB_DIR/$data_name/" || echo "WARNING: Failed to copy $data_name data from $rocm_base/$lib_subdir/$data_name"
+                found=true
+            fi
+        done
+    done
+    if [ "$found" = false ]; then
+        echo "WARNING: No $data_name library data found in ${ROCM_BASE_DIRS:-/opt/rocm*}/lib{,64}/$data_name"
+    fi
+}
+
 # Package AMD ROCm/HIPBlas libraries
 package_rocm_libs() {
     echo "Packaging ROCm/HIPBlas libraries for BUILD_TYPE=${BUILD_TYPE}..."
@@ -267,27 +311,16 @@ package_rocm_libs() {
         fi
     done
 
-    # Copy rocblas library data (tuning files, TensileLibrary, etc.)
-    local old_nullglob=$(shopt -p nullglob)
-    shopt -s nullglob
-    local rocm_dirs=(/opt/rocm /opt/rocm-*)
-    eval "$old_nullglob"
-    local rocblas_found=false
-    for rocm_base in "${rocm_dirs[@]}"; do
-        for lib_subdir in lib lib64; do
-            if [ -d "$rocm_base/$lib_subdir/rocblas" ]; then
-                echo "Found rocblas data at $rocm_base/$lib_subdir/rocblas"
-                mkdir -p "$TARGET_LIB_DIR/rocblas"
-                cp -arfL "$rocm_base/$lib_subdir/rocblas/"* "$TARGET_LIB_DIR/rocblas/" || echo "WARNING: Failed to copy rocblas data from $rocm_base/$lib_subdir/rocblas"
-                rocblas_found=true
-            fi
-        done
-    done
-    if [ "$rocblas_found" = false ]; then
-        echo "WARNING: No rocblas library data found in /opt/rocm*/lib{,64}/rocblas"
-    fi
+    # Copy rocBLAS and hipBLASLt kernel data (TensileLibrary_*.dat tuning files)
+    # so the bundled libs find their per-arch kernels at runtime instead of
+    # falling back to slow generic code (see copy_rocm_data_dir / issue #10660).
+    copy_rocm_data_dir rocblas
+    copy_rocm_data_dir hipblaslt
 
     # Copy libomp from LLVM (required for ROCm)
+    # Single-line `local x=$(...)` on purpose: masks shopt -p's nonzero exit
+    # (nullglob unset) so it doesn't trip `set -e`.
+    local old_nullglob=$(shopt -p nullglob)
     shopt -s nullglob
     local omp_libs=(/opt/rocm*/lib/llvm/lib/libomp.so*)
     eval "$old_nullglob"
@@ -477,6 +510,7 @@ export -f copy_libs_glob
 export -f is_core_lib
 export -f copy_elf_deps
 export -f sweep_transitive_deps
+export -f copy_rocm_data_dir
 export -f package_cuda_libs
 export -f package_rocm_libs
 export -f package_intel_libs

scripts/changed-backends.js

@@ -209,23 +209,75 @@ function splitByArch(entries) {
   return { multiarch, singlearch };
 }
 
+// GitHub Actions refuses to instantiate a matrix with more than 256 jobs. When
+// it happens the job doesn't error visibly — it hangs forever at "Waiting for
+// pending jobs" and the whole run is marked `failure` while every *other* job
+// stays green (seen on the v4.6.1 tag build, run 28786533892: 268 single-arch
+// entries, zero single-arch jobs ever created). The single-arch list is the
+// one that grows unbounded as backends are added, so we shard it across a
+// fixed number of matrix jobs instead of feeding one oversized matrix.
+//
+// SINGLEARCH_SHARDS MUST equal the number of backend-jobs-singlearch-<n>
+// (and backend-merge-jobs-singlearch-<n>) blocks defined in backend.yml and
+// backend_pr.yml. Bump all three together.
+const SINGLEARCH_SHARDS = 4;
+const GHA_MATRIX_LIMIT = 256;
+
+// Split `arr` into exactly `shards` balanced, contiguous chunks. Earlier chunks
+// absorb the remainder when the length doesn't divide evenly; trailing chunks
+// may be empty when there are fewer entries than shards (those emit a
+// has-backends-singlearch-<n>=false flag so their job is skipped).
+function chunkEqually(arr, shards) {
+  const out = [];
+  const base = Math.floor(arr.length / shards);
+  const rem = arr.length % shards;
+  let idx = 0;
+  for (let i = 0; i < shards; i++) {
+    const size = base + (i < rem ? 1 : 0);
+    out.push(arr.slice(idx, idx + size));
+    idx += size;
+  }
+  return out;
+}
+
+// Emit the sharded single-arch build + merge matrices and their has-* gates.
+// Called with the full or filtered single-arch entry list.
+function emitSinglearchShards(singlearch) {
+  const shards = chunkEqually(singlearch, SINGLEARCH_SHARDS);
+  for (let i = 0; i < SINGLEARCH_SHARDS; i++) {
+    const shard = shards[i];
+    // Fail loudly rather than let GitHub silently drop the overflow: a shard at
+    // or above the limit means SINGLEARCH_SHARDS (and the matching job blocks in
+    // both workflows) need to grow.
+    if (shard.length >= GHA_MATRIX_LIMIT) {
+      throw new Error(
+        `single-arch shard ${i + 1} has ${shard.length} entries (>= ${GHA_MATRIX_LIMIT}, ` +
+        `GitHub's per-matrix job limit). Increase SINGLEARCH_SHARDS in ` +
+        `scripts/changed-backends.js and add matching backend-jobs-singlearch-<n> / ` +
+        `backend-merge-jobs-singlearch-<n> blocks to backend.yml and backend_pr.yml.`
+      );
+    }
+    const merge = computeMergeMatrix(shard);
+    const n = i + 1;
+    fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch-${n}=${shard.length > 0 ? 'true' : 'false'}\n`);
+    fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch-${n}=${merge.include.length > 0 ? 'true' : 'false'}\n`);
+    fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch-${n}=${JSON.stringify({ include: shard })}\n`);
+    fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch-${n}=${JSON.stringify(merge)}\n`);
+  }
+}
+
 function emitFullMatrix() {
   const { multiarch, singlearch } = splitByArch(includes);
   const mergeMatrixMultiarch = computeMergeMatrix(multiarch);
-  const mergeMatrixSinglearch = computeMergeMatrix(singlearch);
   const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false';
-  const hasMergesSinglearch = mergeMatrixSinglearch.include.length > 0 ? 'true' : 'false';
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch=${singlearch.length > 0 ? 'true' : 'false'}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${multiarch.length > 0 ? 'true' : 'false'}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch=${hasMergesSinglearch}\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch=${JSON.stringify({ include: singlearch })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch=${JSON.stringify(mergeMatrixSinglearch)}\n`);
+  emitSinglearchShards(singlearch);
   for (const backend of allBackendPaths.keys()) {
     fs.appendFileSync(process.env.GITHUB_OUTPUT, `${backend}=true\n`);
   }
@@ -249,29 +301,23 @@ function emitFilteredMatrix(changedFiles) {
   console.log("Filtered files Darwin:", filteredDarwin);
 
   const { multiarch, singlearch } = splitByArch(filtered);
-  const hasBackendsSinglearch = singlearch.length > 0 ? 'true' : 'false';
   const hasBackendsMultiarch = multiarch.length > 0 ? 'true' : 'false';
   const hasBackendsDarwin = filteredDarwin.length > 0 ? 'true' : 'false';
-  console.log("Has single-arch backends?:", hasBackendsSinglearch);
+  console.log("Has single-arch backends?:", singlearch.length > 0 ? 'true' : 'false');
   console.log("Has multi-arch backends?:", hasBackendsMultiarch);
   console.log("Has Darwin backends?:", hasBackendsDarwin);
 
   const mergeMatrixMultiarch = computeMergeMatrix(multiarch);
-  const mergeMatrixSinglearch = computeMergeMatrix(singlearch);
   const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false';
-  const hasMergesSinglearch = mergeMatrixSinglearch.include.length > 0 ? 'true' : 'false';
 
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch=${hasBackendsSinglearch}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${hasBackendsMultiarch}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch=${hasMergesSinglearch}\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch=${JSON.stringify({ include: singlearch })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`);
   fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`);
-  fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch=${JSON.stringify(mergeMatrixSinglearch)}\n`);
+  emitSinglearchShards(singlearch);
 
   // Per-backend boolean outputs
   for (const [backend, pathPrefix] of allBackendPaths) {

swagger/docs.go

@@ -2728,6 +2728,22 @@ const docTemplate = `{
                 }
             }
         },
+        "/v1/models/capabilities": {
+            "get": {
+                "tags": [
+                    "models"
+                ],
+                "summary": "List available models enriched with capabilities and input/output modalities.",
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.ModelCapabilitiesResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/v1/rerank": {
             "post": {
                 "tags": [
@@ -5182,6 +5198,52 @@ const docTemplate = `{
                 }
             }
         },
+        "schema.ModelCapabilities": {
+            "type": "object",
+            "properties": {
+                "capabilities": {
+                    "description": "Capabilities are canonical usecase strings (e.g. chat, vision, transcript,\ntts, embeddings, image, video) plus the modifiers \"tools\" and \"thinking\".",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "id": {
+                    "type": "string"
+                },
+                "input_modalities": {
+                    "description": "InputModalities is the subset of {text,image,audio,video} the model accepts.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                },
+                "output_modalities": {
+                    "description": "OutputModalities is the subset of {text,image,audio,video} the model produces.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "schema.ModelCapabilitiesResponse": {
+            "type": "object",
+            "properties": {
+                "data": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.ModelCapabilities"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                }
+            }
+        },
         "schema.ModelLoadRequest": {
             "type": "object",
             "properties": {

swagger/swagger.json

@@ -2725,6 +2725,22 @@
                 }
             }
         },
+        "/v1/models/capabilities": {
+            "get": {
+                "tags": [
+                    "models"
+                ],
+                "summary": "List available models enriched with capabilities and input/output modalities.",
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.ModelCapabilitiesResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/v1/rerank": {
             "post": {
                 "tags": [
@@ -5179,6 +5195,52 @@
                 }
             }
         },
+        "schema.ModelCapabilities": {
+            "type": "object",
+            "properties": {
+                "capabilities": {
+                    "description": "Capabilities are canonical usecase strings (e.g. chat, vision, transcript,\ntts, embeddings, image, video) plus the modifiers \"tools\" and \"thinking\".",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "id": {
+                    "type": "string"
+                },
+                "input_modalities": {
+                    "description": "InputModalities is the subset of {text,image,audio,video} the model accepts.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                },
+                "output_modalities": {
+                    "description": "OutputModalities is the subset of {text,image,audio,video} the model produces.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "schema.ModelCapabilitiesResponse": {
+            "type": "object",
+            "properties": {
+                "data": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.ModelCapabilities"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                }
+            }
+        },
         "schema.ModelLoadRequest": {
             "type": "object",
             "properties": {

swagger/swagger.yaml

@@ -1362,6 +1362,41 @@ definitions:
           $ref: '#/definitions/schema.ToolCall'
         type: array
     type: object
+  schema.ModelCapabilities:
+    properties:
+      capabilities:
+        description: |-
+          Capabilities are canonical usecase strings (e.g. chat, vision, transcript,
+          tts, embeddings, image, video) plus the modifiers "tools" and "thinking".
+        items:
+          type: string
+        type: array
+      id:
+        type: string
+      input_modalities:
+        description: InputModalities is the subset of {text,image,audio,video} the
+          model accepts.
+        items:
+          type: string
+        type: array
+      object:
+        type: string
+      output_modalities:
+        description: OutputModalities is the subset of {text,image,audio,video} the
+          model produces.
+        items:
+          type: string
+        type: array
+    type: object
+  schema.ModelCapabilitiesResponse:
+    properties:
+      data:
+        items:
+          $ref: '#/definitions/schema.ModelCapabilities'
+        type: array
+      object:
+        type: string
+    type: object
   schema.ModelLoadRequest:
     properties:
       model:
@@ -4358,6 +4393,16 @@ paths:
       summary: List and describe the various models available in the API.
       tags:
       - models
+  /v1/models/capabilities:
+    get:
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.ModelCapabilitiesResponse'
+      summary: List available models enriched with capabilities and input/output modalities.
+      tags:
+      - models
   /v1/rerank:
     post:
       parameters: