feat(mlx): Add support for CUDA12, CUDA13, L4T, SBSA and CPU (#8380)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.github/workflows/backend.yml

@@ -104,6 +104,45 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-mlx'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'true'
+            backend: "mlx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-mlx-vlm'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'true'
+            backend: "mlx-vlm"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-mlx-audio'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'true'
+            backend: "mlx-audio"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           # CUDA 12 builds
           - build-type: 'cublas'
             cuda-major-version: "12"
@@ -339,6 +378,45 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "12"
+            cuda-minor-version: "8"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-12-mlx'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "mlx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "12"
+            cuda-minor-version: "8"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-12-mlx-vlm'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "mlx-vlm"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "12"
+            cuda-minor-version: "8"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-12-mlx-audio'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "mlx-audio"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "8"
@@ -587,6 +665,45 @@ jobs:
             backend: "diffusers"
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
+          - build-type: 'l4t'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-cuda-13-arm64-mlx'
+            runs-on: 'ubuntu-24.04-arm'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            ubuntu-version: '2404'
+            backend: "mlx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+          - build-type: 'l4t'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-cuda-13-arm64-mlx-vlm'
+            runs-on: 'ubuntu-24.04-arm'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            ubuntu-version: '2404'
+            backend: "mlx-vlm"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+          - build-type: 'l4t'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-cuda-13-arm64-mlx-audio'
+            runs-on: 'ubuntu-24.04-arm'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            ubuntu-version: '2404'
+            backend: "mlx-audio"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -652,6 +769,45 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-13-mlx'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "mlx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-13-mlx-vlm'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "mlx-vlm"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-13-mlx-audio'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "mlx-audio"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -1058,6 +1214,45 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2204'
+          - build-type: 'l4t'
+            cuda-major-version: "12"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-mlx'
+            runs-on: 'ubuntu-24.04-arm'
+            base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+            skip-drivers: 'true'
+            backend: "mlx"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2204'
+          - build-type: 'l4t'
+            cuda-major-version: "12"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-mlx-vlm'
+            runs-on: 'ubuntu-24.04-arm'
+            base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+            skip-drivers: 'true'
+            backend: "mlx-vlm"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2204'
+          - build-type: 'l4t'
+            cuda-major-version: "12"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-mlx-audio'
+            runs-on: 'ubuntu-24.04-arm'
+            base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+            skip-drivers: 'true'
+            backend: "mlx-audio"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2204'
           # SYCL additional backends
           - build-type: 'intel'
             cuda-major-version: ""

backend/index.yaml

@@ -183,6 +183,15 @@
     - text-to-text
     - LLM
     - MLX
+  capabilities:
+    default: "cpu-mlx"
+    nvidia: "cuda12-mlx"
+    metal: "metal-mlx"
+    nvidia-cuda-12: "cuda12-mlx"
+    nvidia-cuda-13: "cuda13-mlx"
+    nvidia-l4t: "nvidia-l4t-mlx"
+    nvidia-l4t-cuda-12: "nvidia-l4t-mlx"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx"
 - &mlx-vlm
   name: "mlx-vlm"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-vlm"
@@ -200,6 +209,15 @@
     - vision-language
     - LLM
     - MLX
+  capabilities:
+    default: "cpu-mlx-vlm"
+    nvidia: "cuda12-mlx-vlm"
+    metal: "metal-mlx-vlm"
+    nvidia-cuda-12: "cuda12-mlx-vlm"
+    nvidia-cuda-13: "cuda13-mlx-vlm"
+    nvidia-l4t: "nvidia-l4t-mlx-vlm"
+    nvidia-l4t-cuda-12: "nvidia-l4t-mlx-vlm"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx-vlm"
 - &mlx-audio
   name: "mlx-audio"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-audio"
@@ -217,6 +235,15 @@
     - text-to-audio
     - LLM
     - MLX
+  capabilities:
+    default: "cpu-mlx-audio"
+    nvidia: "cuda12-mlx-audio"
+    metal: "metal-mlx-audio"
+    nvidia-cuda-12: "cuda12-mlx-audio"
+    nvidia-cuda-13: "cuda13-mlx-audio"
+    nvidia-l4t: "nvidia-l4t-mlx-audio"
+    nvidia-l4t-cuda-12: "nvidia-l4t-mlx-audio"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx-audio"
 - &rerankers
   name: "rerankers"
   alias: "rerankers"
@@ -678,6 +705,159 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-audio"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-mlx-audio
+## mlx
+- !!merge <<: *mlx
+  name: "cpu-mlx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-mlx"
+  mirrors:
+    - localai/localai-backends:latest-cpu-mlx
+- !!merge <<: *mlx
+  name: "cpu-mlx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-mlx"
+  mirrors:
+    - localai/localai-backends:master-cpu-mlx
+- !!merge <<: *mlx
+  name: "cuda12-mlx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-mlx"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-mlx
+- !!merge <<: *mlx
+  name: "cuda12-mlx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-mlx"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-mlx
+- !!merge <<: *mlx
+  name: "cuda13-mlx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-mlx"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-mlx
+- !!merge <<: *mlx
+  name: "cuda13-mlx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-mlx"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-mlx
+- !!merge <<: *mlx
+  name: "nvidia-l4t-mlx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-mlx"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-mlx
+- !!merge <<: *mlx
+  name: "nvidia-l4t-mlx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-mlx"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-mlx
+- !!merge <<: *mlx
+  name: "cuda13-nvidia-l4t-arm64-mlx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-mlx"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-mlx
+- !!merge <<: *mlx
+  name: "cuda13-nvidia-l4t-arm64-mlx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-mlx"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-mlx
+## mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cpu-mlx-vlm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:latest-cpu-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cpu-mlx-vlm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:master-cpu-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cuda12-mlx-vlm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cuda12-mlx-vlm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cuda13-mlx-vlm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cuda13-mlx-vlm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "nvidia-l4t-mlx-vlm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "nvidia-l4t-mlx-vlm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cuda13-nvidia-l4t-arm64-mlx-vlm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-mlx-vlm
+- !!merge <<: *mlx-vlm
+  name: "cuda13-nvidia-l4t-arm64-mlx-vlm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-mlx-vlm
+## mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cpu-mlx-audio"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-mlx-audio"
+  mirrors:
+    - localai/localai-backends:latest-cpu-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cpu-mlx-audio-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-mlx-audio"
+  mirrors:
+    - localai/localai-backends:master-cpu-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cuda12-mlx-audio"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-mlx-audio"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cuda12-mlx-audio-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-mlx-audio"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cuda13-mlx-audio"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-mlx-audio"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cuda13-mlx-audio-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-mlx-audio"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "nvidia-l4t-mlx-audio"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-mlx-audio"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "nvidia-l4t-mlx-audio-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-mlx-audio"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cuda13-nvidia-l4t-arm64-mlx-audio"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-mlx-audio"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-mlx-audio
+- !!merge <<: *mlx-audio
+  name: "cuda13-nvidia-l4t-arm64-mlx-audio-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-mlx-audio"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-mlx-audio
 - !!merge <<: *kitten-tts
   name: "kitten-tts-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-kitten-tts"

backend/python/mlx-audio/requirements-cpu.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cpu]

backend/python/mlx-audio/requirements-cublas12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda12]

backend/python/mlx-audio/requirements-cublas13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda13]

backend/python/mlx-audio/requirements-l4t12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda12]

backend/python/mlx-audio/requirements-l4t13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda13]

backend/python/mlx-vlm/requirements-cpu.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cpu]

backend/python/mlx-vlm/requirements-cublas12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda12]

backend/python/mlx-vlm/requirements-cublas13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda13]

backend/python/mlx-vlm/requirements-l4t12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda12]

backend/python/mlx-vlm/requirements-l4t13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda13]

backend/python/mlx/requirements-cpu.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cpu]

backend/python/mlx/requirements-cublas12.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda12]

backend/python/mlx/requirements-cublas13.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda13]

backend/python/mlx/requirements-l4t12.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda12]

backend/python/mlx/requirements-l4t13.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda13]