mirror of
https://github.com/mudler/LocalAI.git
synced 2026-05-30 03:25:42 -04:00
66af748332
The turboquant fork rebased past ggml-org/llama.cpp#21962, #22397 and #22838, so common_params_speculative now uses the nested draft/ ngram_simple/ngram_mod layout, server_context_impl exposes model_tgt (not model), and get_media_marker() is provided. The compatibility shims in patch-grpc-server.sh were rewriting the shared grpc-server.cpp to the pre-refactor flat layout, which no longer matches the fork and broke the build (see PR #9912 CI failure). Keep only the fork-specific kv_cache_types[] insertion for the TURBO2_0 / TURBO3_0 / TURBO4_0 enum entries. The dormant LOCALAI_LEGACY_LLAMA_CPP_SPEC #ifdef blocks in backend/cpp/llama-cpp/grpc-server.cpp stay as an escape hatch if a future fork bump regresses. Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
69 lines
2.3 KiB
Bash
Executable File
69 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Patch the shared backend/cpp/llama-cpp/grpc-server.cpp *copy* used by the
|
|
# turboquant build:
|
|
#
|
|
# 1. Augment the kv_cache_types[] allow-list so `LoadModel` accepts the
|
|
# fork-specific `turbo2` / `turbo3` / `turbo4` cache types.
|
|
#
|
|
# Historical context: this script used to also paper over API gaps between the
|
|
# fork and upstream (flat vs nested `common_params_speculative`, missing
|
|
# `get_media_marker()`, `ctx_server.impl->model` vs `model_tgt`, and a
|
|
# LOCALAI_LEGACY_LLAMA_CPP_SPEC compile gate). As of TURBOQUANT_VERSION
|
|
# 2cbfdc62 the fork has rebased past ggml-org/llama.cpp#21962, #22397 and
|
|
# #22838, so the shared grpc-server.cpp compiles unmodified against the fork.
|
|
# Only the fork-specific KV-cache enum entries remain.
|
|
#
|
|
# We patch the *copy* sitting in turboquant-<flavor>-build/, never the original
|
|
# under backend/cpp/llama-cpp/, so the stock llama-cpp build stays compiling
|
|
# against vanilla upstream.
|
|
#
|
|
# Idempotent: skips the insertion if its marker is already present (so re-runs
|
|
# of the same build dir don't double-insert).
|
|
|
|
set -euo pipefail
|
|
|
|
if [[ $# -ne 1 ]]; then
|
|
echo "usage: $0 <grpc-server.cpp>" >&2
|
|
exit 2
|
|
fi
|
|
|
|
SRC=$1
|
|
|
|
if [[ ! -f "$SRC" ]]; then
|
|
echo "grpc-server.cpp not found at $SRC" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if grep -q 'GGML_TYPE_TURBO2_0' "$SRC"; then
|
|
echo "==> $SRC already has TurboQuant cache types, skipping KV allow-list patch"
|
|
else
|
|
echo "==> patching $SRC to allow turbo2/turbo3/turbo4 KV-cache types"
|
|
|
|
# Insert the three TURBO entries right after the first ` GGML_TYPE_Q5_1,`
|
|
# line (the kv_cache_types[] allow-list). Using awk because the builder image
|
|
# does not ship python3, and GNU sed's multi-line `a\` quoting is awkward.
|
|
awk '
|
|
/^ GGML_TYPE_Q5_1,$/ && !done {
|
|
print
|
|
print " // turboquant fork extras - added by patch-grpc-server.sh"
|
|
print " GGML_TYPE_TURBO2_0,"
|
|
print " GGML_TYPE_TURBO3_0,"
|
|
print " GGML_TYPE_TURBO4_0,"
|
|
done = 1
|
|
next
|
|
}
|
|
{ print }
|
|
END {
|
|
if (!done) {
|
|
print "patch-grpc-server.sh: anchor ` GGML_TYPE_Q5_1,` not found" > "/dev/stderr"
|
|
exit 1
|
|
}
|
|
}
|
|
' "$SRC" > "$SRC.tmp"
|
|
mv "$SRC.tmp" "$SRC"
|
|
|
|
echo "==> KV allow-list patch OK"
|
|
fi
|
|
|
|
echo "==> all patches applied"
|