mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
mudler-agentandEttore Di Giacinto
86994fac46
* chore(llama-cpp): bump to a37cc461 and drop the TTS patch Move the llama.cpp pin to a37cc461, the merge of the Flint architecture (ggml-org/llama.cpp#30928). The TTS task patch is gone: upstream now carries SERVER_TASK_TYPE_TTS, server_task::tts_inp, server_task_result_tts, server_context_meta::has_cap_tts and the mtmd gen-audio helpers. The patch is detected as already applied on the new pin. The gRPC server uses the same symbols. The bonsai and turboquant forks still lack them and keep using disable-tts-task.sh, so they are unchanged. The score task patch is still needed: upstream has no SERVER_TASK_TYPE_SCORE. It is regenerated against the new pin and applies with no offset or fuzz. Upstream added SERVER_TASK_TYPE_TRANSCRIBE_RT to server_task::need_logits(), which broke one hunk, so SCORE is added next to it by hand. The C++ backend was not built for this change. Assisted-by: Claude:claude-sonnet-5-5 * feat(gallery): add flint Add Recursive Flint, a 39.8B mixture-of-experts coding model with about 2.5B active parameters, from the Unsloth GGUF repository. It runs on llama.cpp with the Flint architecture support added upstream in ggml-org/llama.cpp#30928. Three quantizations are listed, with the default declaring the others as variants: UD-Q4_K_XL (25 GB), UD-Q2_K_XL (13.6 GB) for small hosts and UD-Q6_K_XL (36 GB) for larger ones. Sampling defaults follow the model card (temperature 1, top_p 0.95, top_k 20). The library file gallery/models/index.yaml is edited and gallery/index.yaml is regenerated from it. Assisted-by: Claude:claude-sonnet-5-5 --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
205 lines
9.6 KiB
Makefile
205 lines
9.6 KiB
Makefile
|
|
LLAMA_VERSION?=a37cc46104eb50900d5ab4a9ba02a0abdac039db
|
|
LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
|
|
|
|
CMAKE_ARGS?=
|
|
BUILD_TYPE?=
|
|
NATIVE?=false
|
|
ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
|
|
TARGET?=--target grpc-server
|
|
JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
|
|
ARCH?=$(shell uname -m)
|
|
|
|
# Shared libs default to OFF: we link static gRPC and the avx/avx2/avx512/fallback
|
|
# variants are fully static. The CPU_ALL_VARIANTS build flips SHARED_LIBS=ON (ggml/llama
|
|
# become shared so the dynamic CPU backends work; gRPC stays static via its imported
|
|
# targets). SHARED_LIBS is a make variable, not an appended -D, so it survives the
|
|
# recursive sub-make into the VARIANT build dir (which re-parses this Makefile) instead
|
|
# of being re-clobbered by a second -DBUILD_SHARED_LIBS=OFF. EXTRA_CMAKE_ARGS is the hook
|
|
# the CPU_ALL_VARIANTS target uses to inject -DGGML_BACKEND_DL/-DGGML_CPU_ALL_VARIANTS.
|
|
SHARED_LIBS?=OFF
|
|
EXTRA_CMAKE_ARGS?=
|
|
CMAKE_ARGS+=-DBUILD_SHARED_LIBS=$(SHARED_LIBS) -DLLAMA_CURL=OFF $(EXTRA_CMAKE_ARGS)
|
|
|
|
CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
|
|
ifeq ($(NATIVE),false)
|
|
CMAKE_ARGS+=-DGGML_NATIVE=OFF -DLLAMA_OPENSSL=OFF
|
|
endif
|
|
# If build type is cublas, then we set -DGGML_CUDA=ON to CMAKE_ARGS automatically
|
|
ifeq ($(BUILD_TYPE),cublas)
|
|
CMAKE_ARGS+=-DGGML_CUDA=ON
|
|
# If build type is openblas then we set -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
|
|
# to CMAKE_ARGS automatically
|
|
else ifeq ($(BUILD_TYPE),openblas)
|
|
CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
|
|
# If build type is clblas (openCL) we set -DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
|
|
else ifeq ($(BUILD_TYPE),clblas)
|
|
CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
|
|
# If it's hipblas we do have also to set CC=/opt/rocm/llvm/bin/clang CXX=/opt/rocm/llvm/bin/clang++
|
|
else ifeq ($(BUILD_TYPE),hipblas)
|
|
ROCM_HOME ?= /opt/rocm
|
|
ROCM_PATH ?= /opt/rocm
|
|
export CXX=$(ROCM_HOME)/llvm/bin/clang++
|
|
export CC=$(ROCM_HOME)/llvm/bin/clang
|
|
AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
|
|
ifeq ($(strip $(AMDGPU_TARGETS)),)
|
|
$(error AMDGPU_TARGETS is empty — set it to a comma-separated list of gfx targets e.g. gfx1100,gfx1101)
|
|
endif
|
|
CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
|
|
else ifeq ($(BUILD_TYPE),vulkan)
|
|
CMAKE_ARGS+=-DGGML_VULKAN=1
|
|
else ifeq ($(OS),Darwin)
|
|
ifeq ($(BUILD_TYPE),)
|
|
BUILD_TYPE=metal
|
|
endif
|
|
ifneq ($(BUILD_TYPE),metal)
|
|
CMAKE_ARGS+=-DGGML_METAL=OFF
|
|
else
|
|
CMAKE_ARGS+=-DGGML_METAL=ON
|
|
CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
|
|
CMAKE_ARGS+=-DGGML_METAL_USE_BF16=ON
|
|
CMAKE_ARGS+=-DGGML_OPENMP=OFF
|
|
endif
|
|
TARGET+=--target ggml-metal
|
|
endif
|
|
|
|
ifeq ($(BUILD_TYPE),sycl_f16)
|
|
CMAKE_ARGS+=-DGGML_SYCL=ON \
|
|
-DCMAKE_C_COMPILER=icx \
|
|
-DCMAKE_CXX_COMPILER=icpx \
|
|
-DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \
|
|
-DCMAKE_CXX_FLAGS="-fsycl" \
|
|
-DGGML_SYCL_F16=ON
|
|
endif
|
|
|
|
ifeq ($(BUILD_TYPE),sycl_f32)
|
|
CMAKE_ARGS+=-DGGML_SYCL=ON \
|
|
-DCMAKE_C_COMPILER=icx \
|
|
-DCMAKE_CXX_COMPILER=icpx \
|
|
-DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \
|
|
-DCMAKE_CXX_FLAGS="-fsycl"
|
|
endif
|
|
|
|
INSTALLED_PACKAGES=$(CURDIR)/../grpc/installed_packages
|
|
INSTALLED_LIB_CMAKE=$(INSTALLED_PACKAGES)/lib/cmake
|
|
ADDED_CMAKE_ARGS=-Dabsl_DIR=${INSTALLED_LIB_CMAKE}/absl \
|
|
-DProtobuf_DIR=${INSTALLED_LIB_CMAKE}/protobuf \
|
|
-Dutf8_range_DIR=${INSTALLED_LIB_CMAKE}/utf8_range \
|
|
-DgRPC_DIR=${INSTALLED_LIB_CMAKE}/grpc \
|
|
-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include
|
|
build-llama-cpp-grpc-server:
|
|
# Conditionally build grpc for the llama backend to use if needed
|
|
ifdef BUILD_GRPC_FOR_BACKEND_LLAMA
|
|
$(MAKE) -C ../../grpc build
|
|
_PROTOBUF_PROTOC=${INSTALLED_PACKAGES}/bin/proto \
|
|
_GRPC_CPP_PLUGIN_EXECUTABLE=${INSTALLED_PACKAGES}/bin/grpc_cpp_plugin \
|
|
PATH="${INSTALLED_PACKAGES}/bin:${PATH}" \
|
|
CMAKE_ARGS="${CMAKE_ARGS} ${ADDED_CMAKE_ARGS}" \
|
|
LLAMA_VERSION=$(LLAMA_VERSION) \
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../$(VARIANT) grpc-server
|
|
else
|
|
echo "BUILD_GRPC_FOR_BACKEND_LLAMA is not defined."
|
|
LLAMA_VERSION=$(LLAMA_VERSION) $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../$(VARIANT) grpc-server
|
|
endif
|
|
|
|
llama-cpp-avx2: llama.cpp
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx2-build
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx2-build purge
|
|
$(info ${GREEN}I llama-cpp build info:avx2${RESET})
|
|
CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) VARIANT="llama-cpp-avx2-build" build-llama-cpp-grpc-server
|
|
cp -rfv $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx2-build/grpc-server llama-cpp-avx2
|
|
|
|
llama-cpp-avx512: llama.cpp
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx512-build
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx512-build purge
|
|
$(info ${GREEN}I llama-cpp build info:avx512${RESET})
|
|
CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) VARIANT="llama-cpp-avx512-build" build-llama-cpp-grpc-server
|
|
cp -rfv $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx512-build/grpc-server llama-cpp-avx512
|
|
|
|
llama-cpp-avx: llama.cpp
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx-build
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx-build purge
|
|
$(info ${GREEN}I llama-cpp build info:avx${RESET})
|
|
CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) VARIANT="llama-cpp-avx-build" build-llama-cpp-grpc-server
|
|
cp -rfv $(CURRENT_MAKEFILE_DIR)/../llama-cpp-avx-build/grpc-server llama-cpp-avx
|
|
|
|
llama-cpp-fallback: llama.cpp
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp $(CURRENT_MAKEFILE_DIR)/../llama-cpp-fallback-build
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../llama-cpp-fallback-build purge
|
|
$(info ${GREEN}I llama-cpp build info:fallback${RESET})
|
|
CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) VARIANT="llama-cpp-fallback-build" build-llama-cpp-grpc-server
|
|
cp -rfv $(CURRENT_MAKEFILE_DIR)/../llama-cpp-fallback-build/grpc-server llama-cpp-fallback
|
|
|
|
# Single-build CPU backend using ggml's CPU_ALL_VARIANTS. Produces ONE grpc-server
|
|
# plus a set of dlopen-able libggml-cpu-*.so (sandybridge/haswell/skylakex/...) that
|
|
# ggml's backend registry selects from at runtime by probing host CPU features.
|
|
# Replaces the avx/avx2/avx512/fallback multi-binary build on x86.
|
|
#
|
|
# CPU_ALL_VARIANTS requires GGML_BACKEND_DL, which requires BUILD_SHARED_LIBS=ON, so we
|
|
# pass SHARED_LIBS=ON and the DL flags as make variables (NOT pre-expanded into the
|
|
# CMAKE_ARGS env string): command-line make variables propagate through every recursive
|
|
# sub-make, so the deepest VARIANT-dir build computes BUILD_SHARED_LIBS=ON consistently.
|
|
# Only ggml/llama go shared - gRPC is found via its static imported targets, so the
|
|
# grpc-server binary keeps static gRPC and only dynamically links ggml.
|
|
#
|
|
# TARGET adds "ggml": the per-microarch backends are runtime-dlopened, not link deps of
|
|
# grpc-server, so they only build because each is an add_dependencies() of the ggml target.
|
|
llama-cpp-cpu-all: llama.cpp
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp $(CURRENT_MAKEFILE_DIR)/../llama-cpp-cpu-all-build
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../llama-cpp-cpu-all-build purge
|
|
$(info ${GREEN}I llama-cpp build info:cpu-all-variants${RESET})
|
|
$(MAKE) SHARED_LIBS=ON EXTRA_CMAKE_ARGS="-DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON" TARGET="--target grpc-server --target ggml" VARIANT="llama-cpp-cpu-all-build" build-llama-cpp-grpc-server
|
|
cp -rfv $(CURRENT_MAKEFILE_DIR)/../llama-cpp-cpu-all-build/grpc-server llama-cpp-cpu-all
|
|
rm -rf ggml-shared-libs && mkdir -p ggml-shared-libs
|
|
find $(CURRENT_MAKEFILE_DIR)/../llama-cpp-cpu-all-build/llama.cpp/build \( -name '*.so*' -o -name '*.dylib' \) -exec cp -av {} ggml-shared-libs/ \;
|
|
@echo "Collected ggml shared backends:" && ls -la ggml-shared-libs/
|
|
|
|
llama-cpp-grpc: llama.cpp
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp $(CURRENT_MAKEFILE_DIR)/../llama-cpp-grpc-build
|
|
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../llama-cpp-grpc-build purge
|
|
$(info ${GREEN}I llama-cpp build info:grpc${RESET})
|
|
CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" TARGET="--target grpc-server --target ggml-rpc-server" $(MAKE) VARIANT="llama-cpp-grpc-build" build-llama-cpp-grpc-server
|
|
cp -rfv $(CURRENT_MAKEFILE_DIR)/../llama-cpp-grpc-build/grpc-server llama-cpp-grpc
|
|
|
|
llama-cpp-rpc-server: llama-cpp-grpc
|
|
cp -rf $(CURRENT_MAKEFILE_DIR)/../llama-cpp-grpc-build/llama.cpp/build/bin/ggml-rpc-server llama-cpp-rpc-server
|
|
|
|
llama.cpp:
|
|
mkdir -p llama.cpp
|
|
cd llama.cpp && \
|
|
git init && \
|
|
git remote add origin $(LLAMA_REPO) && \
|
|
git fetch --all --tags && \
|
|
git checkout -b build $(LLAMA_VERSION) && \
|
|
git submodule update --init --recursive --depth 1 --single-branch
|
|
|
|
llama.cpp/tools/grpc-server: llama.cpp
|
|
mkdir -p llama.cpp/tools/grpc-server
|
|
bash prepare.sh
|
|
|
|
rebuild:
|
|
bash prepare.sh
|
|
rm -rf grpc-server
|
|
$(MAKE) grpc-server
|
|
|
|
package:
|
|
bash package.sh
|
|
|
|
purge:
|
|
rm -rf llama.cpp/build
|
|
rm -rf llama.cpp/tools/grpc-server
|
|
rm -rf grpc-server
|
|
|
|
clean: purge
|
|
rm -rf llama.cpp
|
|
|
|
grpc-server: llama.cpp llama.cpp/tools/grpc-server
|
|
@echo "Building grpc-server with $(BUILD_TYPE) build type and $(CMAKE_ARGS)"
|
|
ifneq (,$(findstring sycl,$(BUILD_TYPE)))
|
|
+bash -c "source $(ONEAPI_VARS); \
|
|
cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release -j $(JOBS) $(TARGET)"
|
|
else
|
|
+cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release -j $(JOBS) $(TARGET)
|
|
endif
|
|
cp llama.cpp/build/bin/grpc-server .
|