From 2ccda5ba928c50ae3f72fb082e5a3001ff0b97f5 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 23 Sep 2026 23:12:38 +0200 Subject: [PATCH] chore: :arrow_up: Update TheTom/llama-cpp-turboquant to `4deec5587b2963af00bdf80884f3337e02eb7d64` (#12154) * :arrow_up: Update TheTom/llama-cpp-turboquant Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(turboquant): patch D512 flash-attn shared memory overflow turboquant 4deec55 added DECL_FATTN_VEC_CASE_D512 for TURBO2_0 and TURBO3_0 V cache types. The D=512 kernel template with these types allocates 65 KB of shared memory, exceeding the 48 KB GPU limit: ptxas error: Entry function uses too much shared data (0x10100 bytes, 0xc000 max) Carry the fix as a patch under backend/cpp/turboquant/patches/ until TheTom/llama-cpp-turboquant#386 is merged upstream. TURBO4_0 (4-bit) does not overflow and is left unchanged. Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/cpp/turboquant/Makefile | 2 +- ...move-d512-turbo23-shared-mem-overflow.patch | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch diff --git a/backend/cpp/turboquant/Makefile b/backend/cpp/turboquant/Makefile index 27a113796..e3482d8db 100644 --- a/backend/cpp/turboquant/Makefile +++ b/backend/cpp/turboquant/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of feature/turboquant-kv-cache on https://github.com/TheTom/llama-cpp-turboquant. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -TURBOQUANT_VERSION?=407f3237bfb3eeaff61546797de3d8c1a96be748 +TURBOQUANT_VERSION?=4deec5587b2963af00bdf80884f3337e02eb7d64 LLAMA_REPO?=https://github.com/TheTom/llama-cpp-turboquant CMAKE_ARGS?= diff --git a/backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch b/backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch new file mode 100644 index 000000000..12a937148 --- /dev/null +++ b/backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch @@ -0,0 +1,18 @@ +diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu +index a93be56..3630d87 100644 +--- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu ++++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu +@@ -5,4 +5,3 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); +-DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); +diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu +index 3c806c2..c8a4d9f 100644 +--- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu ++++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu +@@ -5,4 +5,3 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); +-DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);