diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 118cfa300..9ff53684b 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -3234,9 +3234,10 @@ include: # consumed. Same reason CUDA needs its toolkit in base-image rather than in a # builder image: this is the ds4 shape, not the llama-cpp one. # - # No ROCm entry: upstream has no HIP configuration. No CUDA arm64 or L4T - # entry: upstream documents and validates CUDA on x86 only. Darwin/Metal is in - # the includeDarwin matrix below, built by scripts/build/audio-cpp-darwin.sh. + # ROCm uses upstream's HIP backend and the project-wide ROCm 7.2.1 base. No + # CUDA arm64 or L4T entry: upstream documents and validates CUDA on x86 only. + # Darwin/Metal is in the includeDarwin matrix below, built by + # scripts/build/audio-cpp-darwin.sh. # # No vulkan entry either, though Dockerfile.audio-cpp and the backend Makefile # both handle BUILD_TYPE=vulkan for local builds. Every other vulkan backend @@ -3304,6 +3305,19 @@ include: dockerfile: "./backend/Dockerfile.audio-cpp" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-audio-cpp' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + skip-drivers: 'false' + backend: "audio-cpp" + dockerfile: "./backend/Dockerfile.audio-cpp" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" diff --git a/backend/Dockerfile.audio-cpp b/backend/Dockerfile.audio-cpp index 993e2e3b6..108535d14 100644 --- a/backend/Dockerfile.audio-cpp +++ b/backend/Dockerfile.audio-cpp @@ -6,13 +6,14 @@ ARG APT_PORTS_MIRROR="" # ASR, VAD, diarization, source separation and music generation, wrapped as a # LocalAI gRPC backend. # -# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds, or -# nvidia/cuda:-devel-ubuntu24.04 for cublas builds; both ship apt and -# Ubuntu Noble packages, and the CUDA base additionally provides -# /usr/local/cuda. BUILD_TYPE selects the engine backend in the Makefile: -# "" = portable CPU with all ggml CPU variants, "cublas" -> -# -DENGINE_ENABLE_CUDA=ON, "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin -# (Metal) builds bypass this Dockerfile entirely. +# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds, +# nvidia/cuda:-devel-ubuntu24.04 for cublas builds, or +# rocm/dev-ubuntu-24.04: for hipblas builds. All ship apt and Ubuntu Noble +# packages; the GPU bases also provide their toolkits. BUILD_TYPE selects the +# engine backend in the Makefile: "" = portable CPU with all ggml CPU variants, +# "cublas" -> -DENGINE_ENABLE_CUDA=ON, "hipblas" -> -DENGINE_ENABLE_HIP=ON, +# and "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin (Metal) builds bypass this +# Dockerfile entirely. # # Upstream needs GCC 13 or newer, which ubuntu:24.04 and the CUDA 12/13 # devel-ubuntu24.04 images all provide. @@ -62,7 +63,7 @@ ENV BUILD_TYPE=${BUILD_TYPE} \ APT_MIRROR=${APT_MIRROR} \ APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \ DEBIAN_FRONTEND=noninteractive \ - PATH=/usr/local/cuda/bin:${PATH} + PATH=/opt/rocm/bin:/usr/local/cuda/bin:${PATH} WORKDIR /build @@ -73,7 +74,7 @@ WORKDIR /build # fallback of its own. # # BUILD_TYPE=vulkan additionally needs the loader headers and glslc; both are in -# Noble. The CUDA toolkit for BUILD_TYPE=cublas comes from BASE_IMAGE. +# Noble. The CUDA and ROCm toolkits come from their matching BASE_IMAGE. RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \ sh /usr/local/sbin/apt-mirror && \ apt-get update && \ diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 2c26ea926..cf7b29568 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -77,6 +77,13 @@ endif ifeq ($(BUILD_TYPE),cublas) CMAKE_ARGS += -DENGINE_ENABLE_CUDA=ON "-DCMAKE_CUDA_ARCHITECTURES=$(CUDA_ARCHITECTURES)" +else ifeq ($(BUILD_TYPE),hipblas) + ROCM_HOME ?= /opt/rocm + ROCM_PATH ?= /opt/rocm + export CXX=$(ROCM_HOME)/llvm/bin/clang++ + export CC=$(ROCM_HOME)/llvm/bin/clang + AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201 + CMAKE_ARGS += -DENGINE_ENABLE_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS += -DENGINE_ENABLE_VULKAN=ON else ifeq ($(UNAME_S),Darwin) diff --git a/backend/cpp/audio-cpp/loaded_model.cpp b/backend/cpp/audio-cpp/loaded_model.cpp index 77dc10442..28bed59b3 100644 --- a/backend/cpp/audio-cpp/loaded_model.cpp +++ b/backend/cpp/audio-cpp/loaded_model.cpp @@ -103,6 +103,9 @@ engine::core::BackendType parse_backend_type(const std::string &value) { if (value == "cuda") { return engine::core::BackendType::Cuda; } + if (value == "hip" || value == "rocm") { + return engine::core::BackendType::Hip; + } if (value == "vulkan") { return engine::core::BackendType::Vulkan; } @@ -116,7 +119,7 @@ engine::core::BackendType parse_backend_type(const std::string &value) { return engine::core::BackendType::Cpu; } throw ConfigError("audio-cpp: unknown backend option '" + value + - "'. Known backends: cpu, cuda, vulkan, metal, best"); + "'. Known backends: cpu, cuda, hip, rocm, vulkan, metal, best"); } std::filesystem::path executable_directory() { diff --git a/backend/cpp/audio-cpp/model_options.cpp b/backend/cpp/audio-cpp/model_options.cpp index 7a4b8c277..7c5f8d8fd 100644 --- a/backend/cpp/audio-cpp/model_options.cpp +++ b/backend/cpp/audio-cpp/model_options.cpp @@ -59,6 +59,12 @@ bool starts_with(const std::string &value, const std::string &prefix) { value.compare(0, prefix.size(), prefix) == 0; } +bool is_known_backend(const std::string &value) { + return value == "cpu" || value == "cuda" || value == "hip" || + value == "rocm" || value == "vulkan" || value == "metal" || + value == "best"; +} + } // namespace ParsedOptions parse_model_options(const std::vector &entries) { @@ -107,6 +113,12 @@ ParsedOptions parse_model_options(const std::vector &entries) { } else if (key == "task") { parsed.options.task = value; } else if (key == "backend") { + if (!is_known_backend(value)) { + parsed.error = "audio-cpp: unknown backend option '" + value + + "'. Known backends: cpu, cuda, hip, rocm, " + "vulkan, metal, best"; + return parsed; + } parsed.options.backend = value; } else if (key == "model_spec_override") { parsed.options.model_spec_override = value; diff --git a/backend/cpp/audio-cpp/model_options.h b/backend/cpp/audio-cpp/model_options.h index 9cb4464f8..0a5079e97 100644 --- a/backend/cpp/audio-cpp/model_options.h +++ b/backend/cpp/audio-cpp/model_options.h @@ -17,7 +17,7 @@ struct ModelOptions { std::string family; // Pins the audio.cpp task, overriding RPC-based routing. Empty means route. std::string task; - // ggml backend: cpu, cuda, vulkan, metal, best. + // ggml backend: cpu, cuda, hip (or rocm), vulkan, metal, best. std::string backend = "cpu"; int device = 0; // True once a `device:` entry has been seen. 0 is both the default and a diff --git a/backend/cpp/audio-cpp/model_options_test.cpp b/backend/cpp/audio-cpp/model_options_test.cpp index c3cb2f75a..57abc567f 100644 --- a/backend/cpp/audio-cpp/model_options_test.cpp +++ b/backend/cpp/audio-cpp/model_options_test.cpp @@ -75,6 +75,11 @@ static void test_scalar_options() { check(parse_model_options({"live_idle_timeout_ms:0"}).options.live_idle_timeout_ms == 0, "an explicit 0 turns the live idle limit off rather than reverting to " "the default"); + + check(parse_model_options({"backend:hip"}).error.empty(), + "HIP backend option is accepted"); + check(parse_model_options({"backend:rocm"}).error.empty(), + "ROCm backend alias is accepted"); } // Values containing colons must survive: split on the FIRST colon only. @@ -124,6 +129,8 @@ static void test_errors() { "negative device is rejected"); check(!parse_model_options({"threads:x"}).error.empty(), "non-numeric threads is rejected"); + check(!parse_model_options({"backend:unknown"}).error.empty(), + "unknown compute backend is rejected before model loading"); // Values too large for int must be rejected, not silently wrapped into a // negative device index that then reaches the ggml backend selector. diff --git a/backend/index.yaml b/backend/index.yaml index dfd3dfb48..b3a79e587 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -150,6 +150,7 @@ - audio-transcription - CPU - CUDA + - HIP - Metal # No vulkan key: the vulkan image would carry a Vulkan loader with no Mesa ICD # (see the audio-cpp block in .github/backend-matrix.yml). Pointing a @@ -161,6 +162,7 @@ nvidia: "cuda12-audio-cpp" nvidia-cuda-12: "cuda12-audio-cpp" nvidia-cuda-13: "cuda13-audio-cpp" + amd: "rocm-audio-cpp" metal: "metal-audio-cpp" metal-darwin-arm64: "metal-audio-cpp" - &whispercpp @@ -2136,6 +2138,7 @@ nvidia: "cuda12-audio-cpp-development" nvidia-cuda-12: "cuda12-audio-cpp-development" nvidia-cuda-13: "cuda13-audio-cpp-development" + amd: "rocm-audio-cpp-development" metal: "metal-audio-cpp-development" metal-darwin-arm64: "metal-audio-cpp-development" - !!merge <<: *stablediffusionggml @@ -7131,6 +7134,16 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-audio-cpp" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-audio-cpp +- !!merge <<: *audiocpp + name: "rocm-audio-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-audio-cpp" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-audio-cpp +- !!merge <<: *audiocpp + name: "rocm-audio-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-audio-cpp" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-audio-cpp - !!merge <<: *audiocpp name: "metal-audio-cpp" uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-audio-cpp" diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 1b7e059be..4424c165c 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -68,6 +68,49 @@ var _ = Describe("Runtime capability-based backend selection", func() { Expect(cpuArchitectures).To(ConsistOf("linux/amd64/amd64", "linux/arm64/arm64")) }) + It("keeps the audio.cpp ROCm image connected to the AMD capability", func() { + backends, err := ReadConfigFile[[]*GalleryBackend](filepath.Join("..", "..", "backend", "index.yaml")) + Expect(err).NotTo(HaveOccurred()) + + byName := make(map[string]*GalleryBackend, len(*backends)) + for _, backend := range *backends { + byName[backend.Name] = backend + } + + Expect(byName).To(HaveKey("audio-cpp")) + Expect(byName["audio-cpp"].CapabilitiesMap).To(HaveKeyWithValue("amd", "rocm-audio-cpp")) + Expect(byName).To(HaveKey("rocm-audio-cpp")) + Expect(byName["rocm-audio-cpp"].URI).To(Equal("quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-audio-cpp")) + + type matrixEntry struct { + Backend string `yaml:"backend"` + BuildType string `yaml:"build-type"` + Platforms string `yaml:"platforms"` + TagSuffix string `yaml:"tag-suffix"` + BaseImage string `yaml:"base-image"` + } + type backendMatrix struct { + Include []matrixEntry `yaml:"include"` + } + + matrix, err := ReadConfigFile[backendMatrix](filepath.Join("..", "..", ".github", "backend-matrix.yml")) + Expect(err).NotTo(HaveOccurred()) + + var rocmEntries []matrixEntry + for _, entry := range matrix.Include { + if entry.Backend == "audio-cpp" && entry.BuildType == "hipblas" { + rocmEntries = append(rocmEntries, entry) + } + } + Expect(rocmEntries).To(ConsistOf(matrixEntry{ + Backend: "audio-cpp", + BuildType: "hipblas", + Platforms: "linux/amd64", + TagSuffix: "-gpu-rocm-hipblas-audio-cpp", + BaseImage: "rocm/dev-ubuntu-24.04:7.2.1", + })) + }) + It("ListSystemBackends prefers optimal alias candidate", func() { // Arrange two installed backends sharing the same alias must := func(err error) { Expect(err).NotTo(HaveOccurred()) } diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md index 9d59cb378..f3a7ce4be 100644 --- a/docs/content/features/audio-cpp.md +++ b/docs/content/features/audio-cpp.md @@ -109,7 +109,7 @@ listing the keys that exist, rather than being silently ignored. |---|---|---| | `family:` | read from the GGUF | The audio.cpp family. Optional for a standalone audio.cpp GGUF, which embeds `audiocpp.model_spec.family`. Required for a safetensors file or a package directory. Setting it explicitly also overrides the embedded value. | | `task:` | routed from the RPC | Pins the task. One of `gen`, `tts`, `clon`, `vc`, `svc`, `s2s`, `asr`, `align`, `vad`, `diar`, `sep`, `vdes`, `spk`. A pinned task is honoured exactly: if the family cannot serve it, the request is refused rather than rerouted. | -| `backend:` | `cpu` | ggml compute backend: `cpu`, `cuda`, `vulkan`, `metal`, `best`. Must match the backend image you installed (a `cuda` value needs the CUDA image). | +| `backend:` | `cpu` | ggml compute backend: `cpu`, `cuda`, `hip` (`rocm` is an alias), `vulkan`, `metal`, `best`. Must match the backend image you installed (a `hip` value needs the ROCm image). | | `device:` | `0` | GPU index for the selected compute backend. Non-negative integer. | | `threads:` | runtime default | CPU threads. `0` leaves the choice to the runtime. | | `busy_timeout_ms:` | `0` (unbounded) | Bounds the wait for the model's single inference lane. A request that arrives while a run has already been in flight longer than this fails immediately with `UNAVAILABLE` instead of queueing, and a request that waits this long without the lane freeing up fails the same way. `0` waits indefinitely. | @@ -245,8 +245,11 @@ voice conversion from the same weights. | CPU | linux/amd64, linux/arm64 | | CUDA 12 | linux/amd64 | | CUDA 13 | linux/amd64 | +| ROCm 7.2 | linux/amd64 | | Metal | darwin/arm64 | -There is no ROCm image, because upstream has no HIP build configuration, and no Vulkan -image: it would ship a Vulkan loader with no ICD inside the container. `BUILD_TYPE=vulkan` -still works when building the backend yourself. +The ROCm image includes kernels for these GPU targets: +`gfx908`, `gfx90a`, `gfx942`, `gfx950`, `gfx1030`, `gfx1100`, `gfx1101`, `gfx1102`, +`gfx1151`, `gfx1200` and `gfx1201`. Set the model option to `backend:hip` or +`backend:rocm`. There is no Vulkan image because the container would have no Vulkan ICD. +`BUILD_TYPE=vulkan` still works when you build the backend yourself.