From 878b99384fb82a88e3df482bd2e765855f1053f3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Aug 2026 10:08:47 +0000 Subject: [PATCH] feat(audio-cpp): add ROCm backend image The pinned audio.cpp revision supports HIP, but LocalAI neither builds a ROCm image nor accepts its backend option. AMD hosts therefore fall back to the CPU image. Build and publish the HIP variant, connect it to AMD capability selection, and accept both upstream HIP names. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 20 +++++++-- backend/Dockerfile.audio-cpp | 19 +++++---- backend/cpp/audio-cpp/Makefile | 7 ++++ backend/cpp/audio-cpp/loaded_model.cpp | 5 ++- backend/cpp/audio-cpp/model_options.cpp | 12 ++++++ backend/cpp/audio-cpp/model_options.h | 2 +- backend/cpp/audio-cpp/model_options_test.cpp | 7 ++++ backend/index.yaml | 13 ++++++ core/gallery/backends_test.go | 43 ++++++++++++++++++++ docs/content/features/audio-cpp.md | 11 +++-- 10 files changed, 121 insertions(+), 18 deletions(-) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 118cfa300..9ff53684b 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -3234,9 +3234,10 @@ include: # consumed. Same reason CUDA needs its toolkit in base-image rather than in a # builder image: this is the ds4 shape, not the llama-cpp one. # - # No ROCm entry: upstream has no HIP configuration. No CUDA arm64 or L4T - # entry: upstream documents and validates CUDA on x86 only. Darwin/Metal is in - # the includeDarwin matrix below, built by scripts/build/audio-cpp-darwin.sh. + # ROCm uses upstream's HIP backend and the project-wide ROCm 7.2.1 base. No + # CUDA arm64 or L4T entry: upstream documents and validates CUDA on x86 only. + # Darwin/Metal is in the includeDarwin matrix below, built by + # scripts/build/audio-cpp-darwin.sh. # # No vulkan entry either, though Dockerfile.audio-cpp and the backend Makefile # both handle BUILD_TYPE=vulkan for local builds. Every other vulkan backend @@ -3304,6 +3305,19 @@ include: dockerfile: "./backend/Dockerfile.audio-cpp" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-audio-cpp' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + skip-drivers: 'false' + backend: "audio-cpp" + dockerfile: "./backend/Dockerfile.audio-cpp" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" diff --git a/backend/Dockerfile.audio-cpp b/backend/Dockerfile.audio-cpp index 993e2e3b6..108535d14 100644 --- a/backend/Dockerfile.audio-cpp +++ b/backend/Dockerfile.audio-cpp @@ -6,13 +6,14 @@ ARG APT_PORTS_MIRROR="" # ASR, VAD, diarization, source separation and music generation, wrapped as a # LocalAI gRPC backend. # -# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds, or -# nvidia/cuda:-devel-ubuntu24.04 for cublas builds; both ship apt and -# Ubuntu Noble packages, and the CUDA base additionally provides -# /usr/local/cuda. BUILD_TYPE selects the engine backend in the Makefile: -# "" = portable CPU with all ggml CPU variants, "cublas" -> -# -DENGINE_ENABLE_CUDA=ON, "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin -# (Metal) builds bypass this Dockerfile entirely. +# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds, +# nvidia/cuda:-devel-ubuntu24.04 for cublas builds, or +# rocm/dev-ubuntu-24.04: for hipblas builds. All ship apt and Ubuntu Noble +# packages; the GPU bases also provide their toolkits. BUILD_TYPE selects the +# engine backend in the Makefile: "" = portable CPU with all ggml CPU variants, +# "cublas" -> -DENGINE_ENABLE_CUDA=ON, "hipblas" -> -DENGINE_ENABLE_HIP=ON, +# and "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin (Metal) builds bypass this +# Dockerfile entirely. # # Upstream needs GCC 13 or newer, which ubuntu:24.04 and the CUDA 12/13 # devel-ubuntu24.04 images all provide. @@ -62,7 +63,7 @@ ENV BUILD_TYPE=${BUILD_TYPE} \ APT_MIRROR=${APT_MIRROR} \ APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \ DEBIAN_FRONTEND=noninteractive \ - PATH=/usr/local/cuda/bin:${PATH} + PATH=/opt/rocm/bin:/usr/local/cuda/bin:${PATH} WORKDIR /build @@ -73,7 +74,7 @@ WORKDIR /build # fallback of its own. # # BUILD_TYPE=vulkan additionally needs the loader headers and glslc; both are in -# Noble. The CUDA toolkit for BUILD_TYPE=cublas comes from BASE_IMAGE. +# Noble. The CUDA and ROCm toolkits come from their matching BASE_IMAGE. RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ sh /usr/local/sbin/apt-mirror && \ apt-get update && \ diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 2c26ea926..cf7b29568 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -77,6 +77,13 @@ endif ifeq ($(BUILD_TYPE),cublas) CMAKE_ARGS += -DENGINE_ENABLE_CUDA=ON "-DCMAKE_CUDA_ARCHITECTURES=$(CUDA_ARCHITECTURES)" +else ifeq ($(BUILD_TYPE),hipblas) + ROCM_HOME ?= /opt/rocm + ROCM_PATH ?= /opt/rocm + export CXX=$(ROCM_HOME)/llvm/bin/clang++ + export CC=$(ROCM_HOME)/llvm/bin/clang + AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201 + CMAKE_ARGS += -DENGINE_ENABLE_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS += -DENGINE_ENABLE_VULKAN=ON else ifeq ($(UNAME_S),Darwin) diff --git a/backend/cpp/audio-cpp/loaded_model.cpp b/backend/cpp/audio-cpp/loaded_model.cpp index 77dc10442..28bed59b3 100644 --- a/backend/cpp/audio-cpp/loaded_model.cpp +++ b/backend/cpp/audio-cpp/loaded_model.cpp @@ -103,6 +103,9 @@ engine::core::BackendType parse_backend_type(const std::string &value) { if (value == "cuda") { return engine::core::BackendType::Cuda; } + if (value == "hip" || value == "rocm") { + return engine::core::BackendType::Hip; + } if (value == "vulkan") { return engine::core::BackendType::Vulkan; } @@ -116,7 +119,7 @@ engine::core::BackendType parse_backend_type(const std::string &value) { return engine::core::BackendType::Cpu; } throw ConfigError("audio-cpp: unknown backend option '" + value + - "'. Known backends: cpu, cuda, vulkan, metal, best"); + "'. Known backends: cpu, cuda, hip, rocm, vulkan, metal, best"); } std::filesystem::path executable_directory() { diff --git a/backend/cpp/audio-cpp/model_options.cpp b/backend/cpp/audio-cpp/model_options.cpp index 7a4b8c277..7c5f8d8fd 100644 --- a/backend/cpp/audio-cpp/model_options.cpp +++ b/backend/cpp/audio-cpp/model_options.cpp @@ -59,6 +59,12 @@ bool starts_with(const std::string &value, const std::string &prefix) { value.compare(0, prefix.size(), prefix) == 0; } +bool is_known_backend(const std::string &value) { + return value == "cpu" || value == "cuda" || value == "hip" || + value == "rocm" || value == "vulkan" || value == "metal" || + value == "best"; +} + } // namespace ParsedOptions parse_model_options(const std::vector &entries) { @@ -107,6 +113,12 @@ ParsedOptions parse_model_options(const std::vector &entries) { } else if (key == "task") { parsed.options.task = value; } else if (key == "backend") { + if (!is_known_backend(value)) { + parsed.error = "audio-cpp: unknown backend option '" + value + + "'. Known backends: cpu, cuda, hip, rocm, " + "vulkan, metal, best"; + return parsed; + } parsed.options.backend = value; } else if (key == "model_spec_override") { parsed.options.model_spec_override = value; diff --git a/backend/cpp/audio-cpp/model_options.h b/backend/cpp/audio-cpp/model_options.h index 9cb4464f8..0a5079e97 100644 --- a/backend/cpp/audio-cpp/model_options.h +++ b/backend/cpp/audio-cpp/model_options.h @@ -17,7 +17,7 @@ struct ModelOptions { std::string family; // Pins the audio.cpp task, overriding RPC-based routing. Empty means route. std::string task; - // ggml backend: cpu, cuda, vulkan, metal, best. + // ggml backend: cpu, cuda, hip (or rocm), vulkan, metal, best. std::string backend = "cpu"; int device = 0; // True once a `device:` entry has been seen. 0 is both the default and a diff --git a/backend/cpp/audio-cpp/model_options_test.cpp b/backend/cpp/audio-cpp/model_options_test.cpp index c3cb2f75a..57abc567f 100644 --- a/backend/cpp/audio-cpp/model_options_test.cpp +++ b/backend/cpp/audio-cpp/model_options_test.cpp @@ -75,6 +75,11 @@ static void test_scalar_options() { check(parse_model_options({"live_idle_timeout_ms:0"}).options.live_idle_timeout_ms == 0, "an explicit 0 turns the live idle limit off rather than reverting to " "the default"); + + check(parse_model_options({"backend:hip"}).error.empty(), + "HIP backend option is accepted"); + check(parse_model_options({"backend:rocm"}).error.empty(), + "ROCm backend alias is accepted"); } // Values containing colons must survive: split on the FIRST colon only. @@ -124,6 +129,8 @@ static void test_errors() { "negative device is rejected"); check(!parse_model_options({"threads:x"}).error.empty(), "non-numeric threads is rejected"); + check(!parse_model_options({"backend:unknown"}).error.empty(), + "unknown compute backend is rejected before model loading"); // Values too large for int must be rejected, not silently wrapped into a // negative device index that then reaches the ggml backend selector. diff --git a/backend/index.yaml b/backend/index.yaml index dfd3dfb48..b3a79e587 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -150,6 +150,7 @@ - audio-transcription - CPU - CUDA + - HIP - Metal # No vulkan key: the vulkan image would carry a Vulkan loader with no Mesa ICD # (see the audio-cpp block in .github/backend-matrix.yml). Pointing a @@ -161,6 +162,7 @@ nvidia: "cuda12-audio-cpp" nvidia-cuda-12: "cuda12-audio-cpp" nvidia-cuda-13: "cuda13-audio-cpp" + amd: "rocm-audio-cpp" metal: "metal-audio-cpp" metal-darwin-arm64: "metal-audio-cpp" - &whispercpp @@ -2136,6 +2138,7 @@ nvidia: "cuda12-audio-cpp-development" nvidia-cuda-12: "cuda12-audio-cpp-development" nvidia-cuda-13: "cuda13-audio-cpp-development" + amd: "rocm-audio-cpp-development" metal: "metal-audio-cpp-development" metal-darwin-arm64: "metal-audio-cpp-development" - !!merge <<: *stablediffusionggml @@ -7131,6 +7134,16 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-audio-cpp" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-audio-cpp +- !!merge <<: *audiocpp + name: "rocm-audio-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-audio-cpp" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-audio-cpp +- !!merge <<: *audiocpp + name: "rocm-audio-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-audio-cpp" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-audio-cpp - !!merge <<: *audiocpp name: "metal-audio-cpp" uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-audio-cpp" diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 1b7e059be..4424c165c 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -68,6 +68,49 @@ var _ = Describe("Runtime capability-based backend selection", func() { Expect(cpuArchitectures).To(ConsistOf("linux/amd64/amd64", "linux/arm64/arm64")) }) + It("keeps the audio.cpp ROCm image connected to the AMD capability", func() { + backends, err := ReadConfigFile[[]*GalleryBackend](filepath.Join("..", "..", "backend", "index.yaml")) + Expect(err).NotTo(HaveOccurred()) + + byName := make(map[string]*GalleryBackend, len(*backends)) + for _, backend := range *backends { + byName[backend.Name] = backend + } + + Expect(byName).To(HaveKey("audio-cpp")) + Expect(byName["audio-cpp"].CapabilitiesMap).To(HaveKeyWithValue("amd", "rocm-audio-cpp")) + Expect(byName).To(HaveKey("rocm-audio-cpp")) + Expect(byName["rocm-audio-cpp"].URI).To(Equal("quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-audio-cpp")) + + type matrixEntry struct { + Backend string `yaml:"backend"` + BuildType string `yaml:"build-type"` + Platforms string `yaml:"platforms"` + TagSuffix string `yaml:"tag-suffix"` + BaseImage string `yaml:"base-image"` + } + type backendMatrix struct { + Include []matrixEntry `yaml:"include"` + } + + matrix, err := ReadConfigFile[backendMatrix](filepath.Join("..", "..", ".github", "backend-matrix.yml")) + Expect(err).NotTo(HaveOccurred()) + + var rocmEntries []matrixEntry + for _, entry := range matrix.Include { + if entry.Backend == "audio-cpp" && entry.BuildType == "hipblas" { + rocmEntries = append(rocmEntries, entry) + } + } + Expect(rocmEntries).To(ConsistOf(matrixEntry{ + Backend: "audio-cpp", + BuildType: "hipblas", + Platforms: "linux/amd64", + TagSuffix: "-gpu-rocm-hipblas-audio-cpp", + BaseImage: "rocm/dev-ubuntu-24.04:7.2.1", + })) + }) + It("ListSystemBackends prefers optimal alias candidate", func() { // Arrange two installed backends sharing the same alias must := func(err error) { Expect(err).NotTo(HaveOccurred()) } diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md index 9d59cb378..f3a7ce4be 100644 --- a/docs/content/features/audio-cpp.md +++ b/docs/content/features/audio-cpp.md @@ -109,7 +109,7 @@ listing the keys that exist, rather than being silently ignored. |---|---|---| | `family:` | read from the GGUF | The audio.cpp family. Optional for a standalone audio.cpp GGUF, which embeds `audiocpp.model_spec.family`. Required for a safetensors file or a package directory. Setting it explicitly also overrides the embedded value. | | `task:` | routed from the RPC | Pins the task. One of `gen`, `tts`, `clon`, `vc`, `svc`, `s2s`, `asr`, `align`, `vad`, `diar`, `sep`, `vdes`, `spk`. A pinned task is honoured exactly: if the family cannot serve it, the request is refused rather than rerouted. | -| `backend:` | `cpu` | ggml compute backend: `cpu`, `cuda`, `vulkan`, `metal`, `best`. Must match the backend image you installed (a `cuda` value needs the CUDA image). | +| `backend:` | `cpu` | ggml compute backend: `cpu`, `cuda`, `hip` (`rocm` is an alias), `vulkan`, `metal`, `best`. Must match the backend image you installed (a `hip` value needs the ROCm image). | | `device:` | `0` | GPU index for the selected compute backend. Non-negative integer. | | `threads:` | runtime default | CPU threads. `0` leaves the choice to the runtime. | | `busy_timeout_ms:` | `0` (unbounded) | Bounds the wait for the model's single inference lane. A request that arrives while a run has already been in flight longer than this fails immediately with `UNAVAILABLE` instead of queueing, and a request that waits this long without the lane freeing up fails the same way. `0` waits indefinitely. | @@ -245,8 +245,11 @@ voice conversion from the same weights. | CPU | linux/amd64, linux/arm64 | | CUDA 12 | linux/amd64 | | CUDA 13 | linux/amd64 | +| ROCm 7.2 | linux/amd64 | | Metal | darwin/arm64 | -There is no ROCm image, because upstream has no HIP build configuration, and no Vulkan -image: it would ship a Vulkan loader with no ICD inside the container. `BUILD_TYPE=vulkan` -still works when building the backend yourself. +The ROCm image includes kernels for these GPU targets: +`gfx908`, `gfx90a`, `gfx942`, `gfx950`, `gfx1030`, `gfx1100`, `gfx1101`, `gfx1102`, +`gfx1151`, `gfx1200` and `gfx1201`. Set the model option to `backend:hip` or +`backend:rocm`. There is no Vulkan image because the container would have no Vulkan ICD. +`BUILD_TYPE=vulkan` still works when you build the backend yourself.