fix(fish-speech): support CUDA 13 on arm64

Keep the relocated upstream source importable, select CUDA 13 PyTorch wheels instead of the aarch64 CPU fallback, and decode reference audio without torchcodec, which has no Linux arm64 wheels.

Assisted-by: Codex:gpt-5

.docker/llama-cpp-build-target.sh

@@ -8,8 +8,15 @@ build_type=${2-}
 # ggml-cpu/arch/x86/repack.cpp at -march=sapphirerapids: the job sits on that one
 # translation unit until GitHub kills it at 6h. gcc builds the same file in
 # seconds, so only the SYCL images have to give up the CPU variant matrix.
+#
+# ROCm runs out of the same 6h budget for a different reason: volume, not a
+# stall. hipcc compiles ggml's HIP kernels once per entry in AMDGPU_TARGETS,
+# which is eleven architectures (gfx908 through gfx1201), and the CPU variant
+# matrix lands on top of that. The job built in 2h27m before it was added and
+# has been killed at exactly 6h00m on every run since, so no ROCm llama-cpp
+# image has been published since 2026-08-01.
 case "$build_type" in
-  sycl*)
+  sycl*|hipblas*)
     echo llama-cpp-fallback
     exit 0
     ;;

backend/cpp/ds4/CMakeLists.txt

@@ -69,7 +69,15 @@ target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 
 set(DS4_OBJS "${DS4_DIR}/ds4.o")
 if(DS4_GPU STREQUAL "cuda")
-    list(APPEND DS4_OBJS "${DS4_DIR}/ds4_cuda.o")
+    list(APPEND DS4_OBJS
+        "${DS4_DIR}/ds4_cuda.o"
+        "${DS4_DIR}/cuda/mmq/ds4_ggml_stubs.o"
+        "${DS4_DIR}/cuda/mmq/ds4_mmq.o"
+        "${DS4_DIR}/cuda/mmq/ds4_mmq_d2r.o"
+        "${DS4_DIR}/cuda/mmq/quantize.o"
+        "${DS4_DIR}/cuda/mmq/mmid.o"
+        "${DS4_DIR}/cuda/mmq/mmvq.o"
+        "${DS4_DIR}/cuda/mmq/ds4_repack.o")
 elseif(DS4_GPU STREQUAL "metal")
     list(APPEND DS4_OBJS "${DS4_DIR}/ds4_metal.o")
 elseif(DS4_GPU STREQUAL "cpu")

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
+# Upstream pin lives below as DS4_VERSION?=b7e9f0091139999b6c070a57590c447c5741da5c
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
+DS4_VERSION?=b7e9f0091139999b6c070a57590c447c5741da5c
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -23,7 +23,9 @@ CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release
 # are shared by every GPU mode, so append them unconditionally below.
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DDS4_GPU=cuda
-    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \
+        cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \
+        cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
 else ifeq ($(UNAME_S),Darwin)
     CMAKE_ARGS += -DDS4_GPU=metal
     DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
@@ -55,7 +57,7 @@ ds4:
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
 ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C ds4 ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+	+$(MAKE) -C ds4 $(DS4_OBJ_TARGET)
 else ifeq ($(UNAME_S),Darwin)
 	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,30 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=a42b8187caff02c570c28e19e4dc2b1d7f55ed14
+VLLM_CPP_VERSION?=9d1fad3cde0acb95eb0bb0a1025f40a0eb614147
+
+# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
+# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
+# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships
+# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link.
+#
+# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past
+# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is
+# load-bearing, not incidental:
+#
+#   pin >= 89c46aeb, MLX on   -> 99.1% of MLX-LM  (gated: prefill only)
+#   pin <  89c46aeb, MLX on   -> ~51%             (ungated: it also takes decode)
+#
+# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly,
+# because the provider pays an mx::eval sync plus an output memcpy per call and
+# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the
+# good half. So if this pin is ever moved BACKWARDS, this default must go with it.
+VLLM_CPP_MLX?=on
+MLX_VERSION?=0.29.4
+MLX_VENV?=$(abspath ./mlx-venv)
+# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv
+# target has run, and the interpreter version in the path varies per runner.
+MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx)
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -49,6 +72,23 @@ else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
 else ifeq ($(BUILD_TYPE),metal)
 	CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
+	# The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it
+	# is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's
+	# position is that it must earn that cost by measurement. It does, on the
+	# only hardware this build targets: measured on an Apple M4 against the
+	# native MSL GEMM in the SAME binary (arms toggled by
+	# VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to
+	# 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory
+	# and bit-identical output on every parity shape. See vllm.cpp
+	# docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4".
+	#
+	# MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own
+	# kernel, because MLX has no paged-KV primitive at all.
+	#
+	# Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower).
+	ifeq ($(VLLM_CPP_MLX),on)
+		MLX_ENABLED=1
+	endif
 else
 	CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
 endif
@@ -68,10 +108,35 @@ sources/vllm.cpp:
 	git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
 	git checkout FETCH_HEAD
 
-$(LIB): sources/vllm.cpp
+ifeq ($(MLX_ENABLED),1)
+# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than
+# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so
+# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel.
+MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp
+MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT)
+
+$(MLX_STAMP):
+	@if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \
+		python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \
+	fi
+	"$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)"
+	@# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe
+	@# before running its first line, so the glob would still be unmatched here.
+	@# Every later use (the cmake args, package.sh) expands after this target has
+	@# completed, where $(MLX_ROOT) does resolve.
+	@root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \
+	test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \
+		{ echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; }
+	touch $@
+else
+MLX_STAMP=
+MLX_CMAKE_ARGS=
+endif
+
+$(LIB): sources/vllm.cpp $(MLX_STAMP)
 	mkdir -p build && \
 	cd build && \
-	cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
+	cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
 	cmake --build . --config Release -j$(JOBS) --target vllm_shared
 	cp -fL build/$(LIB) ./$(LIB)
 
@@ -79,12 +144,12 @@ vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
 	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
 
 package: vllm-cpp
-	bash package.sh
+	MLX_ROOT="$(MLX_ROOT)" bash package.sh
 
 build: package
 
 clean: purge
-	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
+	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)"
 
 purge:
 	rm -rf build

backend/go/vllm-cpp/README.md

@@ -41,5 +41,50 @@ options:
 - max_num_seqs:16
 ```
 
+## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill)
+
+`BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM
+(`VLLM_CPP_MLX=on`, the default here). It is on because upstream now SHAPE-GATES
+it to prefill; it was briefly off in this branch's history, and that was correct
+at the time for an ungated provider.
+
+The gate matters more than the flag. MLX's steel GEMM wins prefill but loses
+decode, because the provider pays an `mx::eval` synchronisation plus an output
+memcpy on every call and decode makes ~112 calls *per token*. Measured on an
+Apple M4, Qwen3-1.7B-bf16 warm at p=512 g=128:
+
+| configuration | prefill TTFT | warm throughput |
+|---|--:|--:|
+| MLX **gated to prefill** (pin >= 89c46aeb) | **524.5 ms** | **24.37 tok/s, 97.6% of MLX-LM** |
+| MLX ungated (older pins) | 537 ms | 12.7 tok/s |
+| MLX off | 602 ms | 23.9 tok/s, 95.9% |
+
+Ratios are against an MLX-LM baseline measured INTERLEAVED with ours over four
+ABBA blocks (its spread 0.34%, ours 0.12%). An earlier revision of this file
+claimed 99.1%; that used a two-run MLX-LM baseline containing an outlier and
+overstated us by about 1.5 points.
+
+**`VLLM_CPP_VERSION` and this flag are coupled.** Moving the pin back before
+`89c46aeb` while leaving `VLLM_CPP_MLX=on` would take the middle row — roughly
+half throughput. If you roll the pin back, roll the default back with it.
+
+One caveat: MLX's GEMM is not bit-identical to the native kernel, so an MLX build
+produces a different greedy sequence than a non-MLX one. That is a property of the
+provider, not of the gate, and it predates this packaging. Full disposition in
+vllm.cpp `docs/BENCHMARKS.md`.
+
+Build knobs:
+
+- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, and
+  96.4% of MLX-LM instead of 99.1%.
+- `MLX_VERSION` pins the wheel (default `0.29.4`). MLX is consumed as the
+  prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a
+  full Xcode the macOS runners do not have.
+
+Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into
+`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib`
+(re-signing it, since `install_name_tool` invalidates the signature). The
+metallib must stay beside `libmlx.dylib`: MLX looks for it there.
+
 Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=<model>` (and
 optionally `VLLM_CPP_LIBRARY=<libvllm path>`) to enable the e2e specs.

backend/go/vllm-cpp/package.sh

@@ -43,6 +43,50 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
     cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
 elif [ $(uname -s) = "Darwin" ]; then
     echo "Detected Darwin"
+    # Vendor the optional MLX GEMM provider, when libvllm was built against it.
+    # Three facts drive every line below, each verified on an Apple M4 before it
+    # was written:
+    #   1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its
+    #      build-time LC_RPATH points inside the build venv. That path does not
+    #      exist on a user's machine, so it must become @loader_path/lib.
+    #   2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two
+    #      files have to land in the same directory or every Metal op dies with
+    #      "Failed to load the default metallib".
+    #   3. install_name_tool invalidates the code signature, and macOS refuses to
+    #      load an arm64 image whose signature does not match, so the patched
+    #      library must be re-signed ad-hoc afterwards.
+    if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then
+        MLX_LIB_DIR="${MLX_ROOT}/lib"
+        if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then
+            echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2
+            exit 1
+        fi
+        echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR"
+        cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/"
+        cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/"
+        # MLX is MIT and we redistribute its binaries, so its license ships with
+        # them. mlx-metal is the wheel carrying the dylib and the metallib.
+        MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        if [ -z "$MLX_LICENSE" ]; then
+            MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        fi
+        if [ -z "$MLX_LICENSE" ]; then
+            echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2
+            exit 1
+        fi
+        cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx"
+        # Drop every build-tree rpath, then point at the packaged copy.
+        otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do
+            install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true
+        done
+        install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib"
+        codesign -f -s - "$CURDIR/package/libvllm.dylib"
+        # A broken rpath must fail the BUILD, not the user's first inference.
+        if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then
+            echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2
+            exit 1
+        fi
+    fi
 else
     echo "Error: Could not detect architecture"
     exit 1

backend/python/fish-speech/Makefile

@@ -8,8 +8,13 @@ run: fish-speech
 	bash run.sh
 	@echo "fish-speech run."
 
+.PHONY: test-unit
+test-unit:
+	python3 -m unittest -v prepare_upstream_test.py
+	bash run_test.sh
+
 .PHONY: test
-test: fish-speech
+test: fish-speech test-unit
 	@echo "Testing fish-speech..."
 	bash test.sh
 	@echo "fish-speech tested."

backend/python/fish-speech/install.sh

@@ -44,6 +44,13 @@ fi
 # It requires native portaudio libs which aren't available on all build environments.
 sed -i.bak '/"pyaudio"/d' "${FISH_SPEECH_DIR}/pyproject.toml"
 
+# CUDA 13 has no torch 2.8 wheels, so fish-speech's exact upstream pin would
+# make pip select the CPU-only aarch64 wheel from PyPI. Prepare the cloned tree
+# before resolving it, and use soundfile for reference audio because torchcodec
+# does not publish Linux aarch64 wheels.
+python3 "${backend_dir}/prepare_upstream.py" "${FISH_SPEECH_DIR}" \
+    --cuda-major "${CUDA_MAJOR_VERSION:-}"
+
 # Install fish-speech deps from source (without the package itself since we use PYTHONPATH)
 ensureVenv
 if [ "x${USE_PIP}" == "xtrue" ]; then

backend/python/fish-speech/prepare_upstream.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+import argparse
+from pathlib import Path
+
+
+TORCH_28 = '"torch==2.8.0"'
+TORCH_29 = '"torch==2.9.1"'
+TORCHAUDIO_28 = '"torchaudio==2.8.0"'
+TORCHAUDIO_29 = '"torchaudio==2.9.1"'
+TORCHAUDIO_LOAD = (
+    "        waveform, original_sr = "
+    "torchaudio.load(reference_audio, backend=self.backend)"
+)
+SOUNDFILE_LOAD = "\n".join(
+    (
+        "        import soundfile as _sf",
+        "        import torch as _torch",
+        "",
+        "        data, original_sr = _sf.read(",
+        '            reference_audio, dtype="float32", always_2d=True',
+        "        )",
+        "        waveform = _torch.from_numpy(data.T.copy())",
+    )
+)
+
+
+def patch_cuda13_dependencies(pyproject: Path) -> None:
+    content = pyproject.read_text()
+    if (
+        TORCH_28 not in content
+        and TORCHAUDIO_28 not in content
+        and TORCH_29 in content
+        and TORCHAUDIO_29 in content
+    ):
+        return
+    if TORCH_28 not in content or TORCHAUDIO_28 not in content:
+        raise RuntimeError("fish-speech's torch 2.8 dependency pins have changed")
+
+    content = content.replace(TORCH_28, TORCH_29)
+    content = content.replace(TORCHAUDIO_28, TORCHAUDIO_29)
+    pyproject.write_text(content)
+
+
+def patch_reference_loader(loader: Path) -> None:
+    content = loader.read_text()
+    if TORCHAUDIO_LOAD not in content and content.count(SOUNDFILE_LOAD) == 1:
+        return
+    if content.count(TORCHAUDIO_LOAD) != 1:
+        raise RuntimeError("fish-speech's torchaudio.load call has changed")
+
+    loader.write_text(content.replace(TORCHAUDIO_LOAD, SOUNDFILE_LOAD))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("source", type=Path)
+    parser.add_argument("--cuda-major")
+    args = parser.parse_args()
+
+    if args.cuda_major == "13":
+        patch_cuda13_dependencies(args.source / "pyproject.toml")
+    patch_reference_loader(
+        args.source / "fish_speech/inference_engine/reference_loader.py"
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/python/fish-speech/prepare_upstream_test.py

@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: MIT
+
+import importlib.util
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+
+
+MODULE_PATH = Path(__file__).with_name("prepare_upstream.py")
+
+
+def load_prepare_upstream():
+    if not MODULE_PATH.exists():
+        raise AssertionError("prepare_upstream.py is missing")
+    spec = importlib.util.spec_from_file_location("prepare_upstream", MODULE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class FakeAudioData:
+    @property
+    def T(self):
+        return self
+
+    def copy(self):
+        return "channels-first"
+
+
+class PrepareUpstreamTests(unittest.TestCase):
+    def test_cuda13_dependencies_follow_available_pytorch_wheels(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            pyproject = Path(tmp) / "pyproject.toml"
+            pyproject.write_text(
+                'dependencies = [\n    "torch==2.8.0",\n    "torchaudio==2.8.0",\n]\n'
+                'stable = [\n    "torch==2.8.0",\n    "torchaudio",\n]\n'
+            )
+
+            prepare_upstream.patch_cuda13_dependencies(pyproject)
+
+            self.assertEqual(
+                pyproject.read_text(),
+                'dependencies = [\n    "torch==2.9.1",\n    "torchaudio==2.9.1",\n]\n'
+                'stable = [\n    "torch==2.9.1",\n    "torchaudio",\n]\n',
+            )
+
+    def test_reference_audio_uses_soundfile_without_torchcodec(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            loader = Path(tmp) / "reference_loader.py"
+            loader.write_text(
+                "class ReferenceLoader:\n"
+                "    def load_audio(self, reference_audio):\n"
+                "        waveform, original_sr = torchaudio.load(reference_audio, backend=self.backend)\n"
+                "        return waveform, original_sr\n"
+            )
+            prepare_upstream.patch_reference_loader(loader)
+
+            calls = []
+            fake_soundfile = types.SimpleNamespace(
+                read=lambda source, **kwargs: (
+                    calls.append((source, kwargs)) or FakeAudioData(),
+                    24000,
+                )
+            )
+            fake_torch = types.SimpleNamespace(
+                from_numpy=lambda data: ("tensor", data),
+            )
+            previous_soundfile = sys.modules.get("soundfile")
+            previous_torch = sys.modules.get("torch")
+            sys.modules["soundfile"] = fake_soundfile
+            sys.modules["torch"] = fake_torch
+            try:
+                namespace = {"torchaudio": None}
+                exec(compile(loader.read_text(), str(loader), "exec"), namespace)
+                instance = namespace["ReferenceLoader"]()
+                instance.backend = "soundfile"
+
+                waveform, sample_rate = instance.load_audio("voice.wav")
+            finally:
+                if previous_soundfile is None:
+                    del sys.modules["soundfile"]
+                else:
+                    sys.modules["soundfile"] = previous_soundfile
+                if previous_torch is None:
+                    del sys.modules["torch"]
+                else:
+                    sys.modules["torch"] = previous_torch
+
+            self.assertEqual(waveform, ("tensor", "channels-first"))
+            self.assertEqual(sample_rate, 24000)
+            self.assertEqual(
+                calls,
+                [("voice.wav", {"dtype": "float32", "always_2d": True})],
+            )
+
+    def test_reference_loader_drift_fails_the_build(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            loader = Path(tmp) / "reference_loader.py"
+            loader.write_text("def load_audio():\n    pass\n")
+
+            with self.assertRaisesRegex(RuntimeError, "torchaudio.load call"):
+                prepare_upstream.patch_reference_loader(loader)
+
+    def test_preparation_can_be_repeated(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            pyproject = Path(tmp) / "pyproject.toml"
+            pyproject.write_text(
+                'dependencies = ["torch==2.8.0", "torchaudio==2.8.0"]\n'
+            )
+            loader = Path(tmp) / "reference_loader.py"
+            loader.write_text(
+                "def load_audio(reference_audio):\n"
+                "        waveform, original_sr = torchaudio.load(reference_audio, backend=self.backend)\n"
+            )
+
+            prepare_upstream.patch_cuda13_dependencies(pyproject)
+            prepare_upstream.patch_reference_loader(loader)
+            try:
+                prepare_upstream.patch_cuda13_dependencies(pyproject)
+                prepare_upstream.patch_reference_loader(loader)
+            except RuntimeError as err:
+                self.fail(f"preparation is not idempotent: {err}")
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/fish-speech/requirements-cublas13.txt

@@ -1,3 +1,3 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
+torch==2.9.1+cu130
+torchaudio==2.9.1

backend/python/fish-speech/requirements-l4t13.txt

@@ -1,3 +1,3 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
+torch==2.9.1+cu130
+torchaudio==2.9.1

backend/python/fish-speech/run.sh

@@ -6,4 +6,8 @@ else
     source $backend_dir/../common/libbackend.sh
 fi
 
-startBackend $@
+# Editable installs record their build-time absolute source path, which becomes
+# stale when the backend is relocated under /backends at install time.
+export PYTHONPATH="${EDIR}/fish-speech-src${PYTHONPATH:+:${PYTHONPATH}}"
+
+startBackend "$@"

backend/python/fish-speech/run_test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(cd "$(dirname "$0")" && pwd)
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+mkdir -p "$work/backend/common" "$work/backend/fish-speech-src"
+cp "$backend_dir/run.sh" "$work/backend/run.sh"
+
+cat > "$work/backend/common/libbackend.sh" <<'EOF'
+EDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+startBackend() {
+    printf '%s\n' "$PYTHONPATH"
+}
+EOF
+
+actual=$(PYTHONPATH=/existing/path bash "$work/backend/run.sh")
+expected="$work/backend/fish-speech-src:/existing/path"
+
+if [ "$actual" != "$expected" ]; then
+    printf 'expected PYTHONPATH %s, got %s\n' "$expected" "$actual" >&2
+    exit 1
+fi
+
+echo "PASS: relocated fish-speech source is importable"

core/http/react-ui/package-lock.json

@@ -24,7 +24,7 @@
         "@modelcontextprotocol/sdk": "^1.30.0",
         "dompurify": "^3.4.12",
         "highlight.js": "^11.11.1",
-        "hono": "4.12.25",
+        "hono": "4.12.34",
         "i18next": "^26.0.8",
         "i18next-browser-languagedetector": "^8.2.1",
         "i18next-http-backend": "^3.0.6",
@@ -636,12 +636,12 @@
       }
     },
     "node_modules/@hono/node-server": {
-      "version": "1.19.14",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz",
-      "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.0.tgz",
+      "integrity": "sha512-XovyyCCnBzW+zKu+z/zq8hwNs4KOR5rEMAOxo2f40Q5xoOI37IMm6MIg2COOUtUApo0i6850MTBKH2u4QLGIqg==",
       "license": "MIT",
       "engines": {
-        "node": ">=18.14.1"
+        "node": ">=20"
       },
       "peerDependencies": {
         "hono": "^4"
@@ -2879,9 +2879,9 @@
       "dev": true
     },
     "node_modules/fast-uri": {
-      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz",
-      "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
+      "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
       "funding": [
         {
           "type": "github",
@@ -3435,9 +3435,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.12.25",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz",
-      "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==",
+      "version": "4.12.34",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.34.tgz",
+      "integrity": "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
@@ -4196,9 +4196,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "node_modules/ip-address": {
-      "version": "10.2.0",
-      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
-      "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
+      "version": "10.4.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz",
+      "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==",
       "license": "MIT",
       "engines": {
         "node": ">= 12"
@@ -7138,9 +7138,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "7.28.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
-      "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
+      "version": "7.29.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
+      "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
       "dev": true,
       "license": "MIT",
       "engines": {

core/http/react-ui/package.json

@@ -19,7 +19,7 @@
     "coverage:report": "nyc report"
   },
   "overrides": {
-    "hono": "4.12.25"
+    "hono": "4.12.34"
   },
   "dependencies": {
     "@codemirror/autocomplete": "^6.18.6",
@@ -38,7 +38,7 @@
     "@modelcontextprotocol/sdk": "^1.30.0",
     "dompurify": "^3.4.12",
     "highlight.js": "^11.11.1",
-    "hono": "4.12.25",
+    "hono": "4.12.34",
     "i18next": "^26.0.8",
     "i18next-browser-languagedetector": "^8.2.1",
     "i18next-http-backend": "^3.0.6",

core/services/advisorylock/leader_loop_test.go

@@ -54,62 +54,57 @@ var _ = Describe("RunLeaderLoop", func() {
 				close(done)
 			}()
 
-			// Let it run a bit then cancel
-			time.Sleep(150 * time.Millisecond)
+			Eventually(func() int32 {
+				return atomic.LoadInt32(&callCount)
+			}, 500*time.Millisecond, 10*time.Millisecond).Should(BeNumerically(">=", 1))
 			cancel()
 
-			// RunLeaderLoop should return
 			Eventually(done, 500*time.Millisecond).Should(BeClosed())
-
-			// Record count after cancellation
-			countAfterCancel := atomic.LoadInt32(&callCount)
-			time.Sleep(150 * time.Millisecond)
-			countLater := atomic.LoadInt32(&callCount)
-
-			Expect(countLater).To(Equal(countAfterCancel),
-				"function should stop being called after context cancellation")
 		})
 
 		It("only one leader executes at a time (two concurrent loops)", func() {
 			db := testutil.SetupTestDB()
 			const lockKey int64 = 5002
 
-			var (
-				mu         sync.Mutex
-				maxRunning int32
-				running    int32
-			)
+			var running int32
+			entered := make(chan struct{}, 2)
+			release := make(chan struct{})
+			var releaseOnce sync.Once
 
 			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
+			done := make(chan struct{}, 2)
+			DeferCleanup(func() {
+				cancel()
+				releaseOnce.Do(func() { close(release) })
+			})
 
 			fn := func() {
-				cur := atomic.AddInt32(&running, 1)
-				mu.Lock()
-				if cur > maxRunning {
-					maxRunning = cur
+				atomic.AddInt32(&running, 1)
+				select {
+				case entered <- struct{}{}:
+				default:
 				}
-				mu.Unlock()
-
-				time.Sleep(30 * time.Millisecond)
-
+				<-release
 				atomic.AddInt32(&running, -1)
 			}
 
-			// Start two competing leader loops with the same lock key
-			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
-			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
+			for range 2 {
+				go func() {
+					RunLeaderLoop(ctx, db, lockKey, 1*time.Millisecond, fn)
+					done <- struct{}{}
+				}()
+			}
+
+			Eventually(entered, 500*time.Millisecond).Should(Receive())
+			Consistently(func() int32 {
+				return atomic.LoadInt32(&running)
+			}, 50*time.Millisecond, 5*time.Millisecond).Should(Equal(int32(1)),
+				"expected only the lock holder to run while both loops tick")
 
-			// Let them run for a while
-			time.Sleep(400 * time.Millisecond)
 			cancel()
-
-			mu.Lock()
-			observed := maxRunning
-			mu.Unlock()
-
-			Expect(observed).To(BeNumerically("<=", 1),
-				"expected at most 1 goroutine running the leader function at a time")
+			releaseOnce.Do(func() { close(release) })
+			Eventually(done, 500*time.Millisecond).Should(Receive())
+			Eventually(done, 500*time.Millisecond).Should(Receive())
 		})
 	})
 })

scripts/build/llama-cpp-build-target_test.sh

@@ -29,4 +29,11 @@ assert_target arm64 "" llama-cpp-cpu-all
 assert_target amd64 sycl_f16 llama-cpp-fallback
 assert_target amd64 sycl_f32 llama-cpp-fallback
 
+# ROCm exhausts the same 6h budget through volume rather than a stall: hipcc
+# compiles ggml's HIP kernels once per AMDGPU target, eleven of them, and the
+# CPU variant matrix goes on top. 2h27m before it was added, killed at exactly
+# 6h00m on every run since.
+assert_target amd64 hipblas llama-cpp-fallback
+assert_target arm64 hipblas llama-cpp-fallback
+
 echo "PASS: llama.cpp build target preserves CPU variants where supported"