From 81eaca8768287ce380ea548bcef0b3bf7eab24d1 Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Thu, 24 Sep 2026 08:01:28 +0200 Subject: [PATCH] fix(turboquant): extend D512 flash-attn patch to all turbo V types (#12234) The previous patch only removed DECL_FATTN_VEC_CASE_D512 for turbo2_0 and turbo3_0 V cache types. turbo4_0 also overflows shared memory (0x10100 bytes > 0xc000 max), causing ptxas errors on CUDA 12/13. Additionally, the previous patch was incomplete: it only removed the template instantiations but not the dispatch calls in fattn.cu or the extern declarations in fattn-vec.cuh. This caused linker errors (undefined reference to ggml_cuda_flash_attn_ext_vec_case_d512). This patch removes all three layers for all turbo V types: - Template instance .cu files (DECL_FATTN_VEC_CASE_D512) - Dispatch calls in fattn.cu (FATTN_VEC_CASE_D512) - Extern declarations in fattn-vec.cuh (extern DECL_FATTN_VEC_CASE_D512) Signed-off-by: mudler Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- ...emove-d512-turbo-shared-mem-overflow.patch | 52 +++++++++++++++++++ ...ove-d512-turbo23-shared-mem-overflow.patch | 18 ------- 2 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 backend/cpp/turboquant/patches/0002-remove-d512-turbo-shared-mem-overflow.patch delete mode 100644 backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch diff --git a/backend/cpp/turboquant/patches/0002-remove-d512-turbo-shared-mem-overflow.patch b/backend/cpp/turboquant/patches/0002-remove-d512-turbo-shared-mem-overflow.patch new file mode 100644 index 000000000..7dfb385c3 --- /dev/null +++ b/backend/cpp/turboquant/patches/0002-remove-d512-turbo-shared-mem-overflow.patch @@ -0,0 +1,52 @@ +diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh +index 680fd12..ffd6604 100644 +--- a/ggml/src/ggml-cuda/fattn-vec.cuh ++++ b/ggml/src/ggml-cuda/fattn-vec.cuh +@@ -980,6 +980,3 @@ extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0); + extern DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_F16); + extern DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0); + extern DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_BF16); +-extern DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); +-extern DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); +-extern DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0); +diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu +index 5c614a9..d765cfc 100644 +--- a/ggml/src/ggml-cuda/fattn.cu ++++ b/ggml/src/ggml-cuda/fattn.cu +@@ -507,9 +507,6 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t + FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_F16) + FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) + FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_BF16) +- FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0) +- FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0) +- FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0) + + #ifdef GGML_CUDA_FA_ALL_QUANTS + FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) +diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu +index a93be56..3630d87 100644 +--- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu ++++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu +@@ -5,4 +5,3 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); +-DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); +diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu +index 3c806c2..c8a4d9f 100644 +--- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu ++++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu +@@ -5,4 +5,3 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); +-DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); +diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo4_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo4_0.cu +index 180902f..1646ef0 100644 +--- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo4_0.cu ++++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo4_0.cu +@@ -5,4 +5,3 @@ + DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0); + DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0); + DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0); +-DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0); diff --git a/backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch b/backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch deleted file mode 100644 index 12a937148..000000000 --- a/backend/cpp/turboquant/patches/0002-remove-d512-turbo23-shared-mem-overflow.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu -index a93be56..3630d87 100644 ---- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu -+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu -@@ -5,4 +5,3 @@ - DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); - DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); - DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); --DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0); -diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu -index 3c806c2..c8a4d9f 100644 ---- a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu -+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu -@@ -5,4 +5,3 @@ - DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); - DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); - DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0); --DECL_FATTN_VEC_CASE_D512(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);