From 79b68fdc255a3c2985679e3125e94883e0ecdc20 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 28 Oct 2025 11:05:07 +0100 Subject: [PATCH] feat: add CPU variants for whisper.cpp (#6855) * feat: add CPU variants for whisper.cpp Signed-off-by: Ettore Di Giacinto * Do not build variants on Darwin Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto --- backend/go/whisper/Makefile | 65 +++++++++++++++++++++++++++++------ backend/go/whisper/main.go | 9 ++++- backend/go/whisper/package.sh | 3 +- backend/go/whisper/run.sh | 38 ++++++++++++++++++++ 4 files changed, 103 insertions(+), 12 deletions(-) diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index 932760fd6..0ba080de6 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -9,6 +9,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp WHISPER_CPP_VERSION?=f16c12f3f55f5bd3d6ac8cf2f31ab90a42c884d5 +SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF @@ -57,15 +58,18 @@ sources/whisper.cpp: git checkout $(WHISPER_CPP_VERSION) && \ git submodule update --init --recursive --depth 1 --single-branch -libgowhisper.so: sources/whisper.cpp CMakeLists.txt gowhisper.cpp gowhisper.h - mkdir -p build && \ - cd build && \ - cmake .. $(CMAKE_ARGS) && \ - cmake --build . --config Release -j$(JOBS) && \ - cd .. && \ - mv build/libgowhisper.so ./ +# Detect OS +UNAME_S := $(shell uname -s) -whisper: main.go gowhisper.go libgowhisper.so +# Only build CPU variants on Linux +ifeq ($(UNAME_S),Linux) + VARIANT_TARGETS = libgowhisper-avx.so libgowhisper-avx2.so libgowhisper-avx512.so libgowhisper-fallback.so +else + # On non-Linux (e.g., Darwin), build only fallback variant + VARIANT_TARGETS = libgowhisper-fallback.so +endif + +whisper: main.go gowhisper.go $(VARIANT_TARGETS) CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o whisper ./ package: whisper @@ -73,5 +77,46 @@ package: whisper build: package -clean: - rm -rf libgowhisper.o build whisper +clean: purge + rm -rf libgowhisper*.so sources/whisper.cpp whisper + +purge: + rm -rf build* + +# Build all variants (Linux only) +ifeq ($(UNAME_S),Linux) +libgowhisper-avx.so: sources/whisper.cpp + $(MAKE) purge + $(info ${GREEN}I whisper build info:avx${RESET}) + SO_TARGET=libgowhisper-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" $(MAKE) libgowhisper-custom + rm -rfv build* + +libgowhisper-avx2.so: sources/whisper.cpp + $(MAKE) purge + $(info ${GREEN}I whisper build info:avx2${RESET}) + SO_TARGET=libgowhisper-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) libgowhisper-custom + rm -rfv build* + +libgowhisper-avx512.so: sources/whisper.cpp + $(MAKE) purge + $(info ${GREEN}I whisper build info:avx512${RESET}) + SO_TARGET=libgowhisper-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) libgowhisper-custom + rm -rfv build* +endif + +# Build fallback variant (all platforms) +libgowhisper-fallback.so: sources/whisper.cpp + $(MAKE) purge + $(info ${GREEN}I whisper build info:fallback${RESET}) + SO_TARGET=libgowhisper-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" $(MAKE) libgowhisper-custom + rm -rfv build* + +libgowhisper-custom: CMakeLists.txt gowhisper.cpp gowhisper.h + mkdir -p build-$(SO_TARGET) && \ + cd build-$(SO_TARGET) && \ + cmake .. $(CMAKE_ARGS) && \ + cmake --build . --config Release -j$(JOBS) && \ + cd .. && \ + mv build-$(SO_TARGET)/libgowhisper.so ./$(SO_TARGET) + +all: whisper package diff --git a/backend/go/whisper/main.go b/backend/go/whisper/main.go index 0ca3905fe..794c0a228 100644 --- a/backend/go/whisper/main.go +++ b/backend/go/whisper/main.go @@ -3,6 +3,7 @@ package main // Note: this is started internally by LocalAI and a server is allocated for each model import ( "flag" + "os" "github.com/ebitengine/purego" grpc "github.com/mudler/LocalAI/pkg/grpc" @@ -18,7 +19,13 @@ type LibFuncs struct { } func main() { - gosd, err := purego.Dlopen("./libgowhisper.so", purego.RTLD_NOW|purego.RTLD_GLOBAL) + // Get library name from environment variable, default to fallback + libName := os.Getenv("WHISPER_LIBRARY") + if libName == "" { + libName = "./libgowhisper-fallback.so" + } + + gosd, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL) if err != nil { panic(err) } diff --git a/backend/go/whisper/package.sh b/backend/go/whisper/package.sh index 4e5c4fee2..9173706f2 100755 --- a/backend/go/whisper/package.sh +++ b/backend/go/whisper/package.sh @@ -10,7 +10,8 @@ CURDIR=$(dirname "$(realpath $0)") # Create lib directory mkdir -p $CURDIR/package/lib -cp -avf $CURDIR/whisper $CURDIR/libgowhisper.so $CURDIR/package/ +cp -avf $CURDIR/whisper $CURDIR/package/ +cp -fv $CURDIR/libgowhisper-*.so $CURDIR/package/ cp -fv $CURDIR/run.sh $CURDIR/package/ # Detect architecture and copy appropriate libraries diff --git a/backend/go/whisper/run.sh b/backend/go/whisper/run.sh index 2c23f321a..1af2c0535 100755 --- a/backend/go/whisper/run.sh +++ b/backend/go/whisper/run.sh @@ -1,14 +1,52 @@ #!/bin/bash set -ex +# Get the absolute current dir where the script is located CURDIR=$(dirname "$(realpath $0)") +cd / + +echo "CPU info:" +if [ "$(uname)" != "Darwin" ]; then + grep -e "model\sname" /proc/cpuinfo | head -1 + grep -e "flags" /proc/cpuinfo | head -1 +fi + +LIBRARY="$CURDIR/libgowhisper-fallback.so" + +if [ "$(uname)" != "Darwin" ]; then + if grep -q -e "\savx\s" /proc/cpuinfo ; then + echo "CPU: AVX found OK" + if [ -e $CURDIR/libgowhisper-avx.so ]; then + LIBRARY="$CURDIR/libgowhisper-avx.so" + fi + fi + + if grep -q -e "\savx2\s" /proc/cpuinfo ; then + echo "CPU: AVX2 found OK" + if [ -e $CURDIR/libgowhisper-avx2.so ]; then + LIBRARY="$CURDIR/libgowhisper-avx2.so" + fi + fi + + # Check avx 512 + if grep -q -e "\savx512f\s" /proc/cpuinfo ; then + echo "CPU: AVX512F found OK" + if [ -e $CURDIR/libgowhisper-avx512.so ]; then + LIBRARY="$CURDIR/libgowhisper-avx512.so" + fi + fi +fi + export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH +export WHISPER_LIBRARY=$LIBRARY # If there is a lib/ld.so, use it if [ -f $CURDIR/lib/ld.so ]; then echo "Using lib/ld.so" + echo "Using library: $LIBRARY" exec $CURDIR/lib/ld.so $CURDIR/whisper "$@" fi +echo "Using library: $LIBRARY" exec $CURDIR/whisper "$@" \ No newline at end of file