diff --git a/backend/python/common/libbackend.sh b/backend/python/common/libbackend.sh index 439725212..14172decf 100644 --- a/backend/python/common/libbackend.sh +++ b/backend/python/common/libbackend.sh @@ -398,40 +398,80 @@ function ensureVenv() { function runProtogen() { ensureVenv - # Match grpcio-tools to the grpcio already installed by the backend's - # requirements. grpcio and grpcio-tools are released in lockstep, and the - # protoc that grpcio-tools bundles stamps a Protobuf "gencode" version into - # backend_pb2.py. Left unpinned, `uv pip install grpcio-tools` pulls the - # newest release, whose newer gencode (e.g. 7.35.0) trips Protobuf's - # runtime >= gencode guarantee at import time when a backend caps the - # protobuf runtime lower (vLLM pins it to 6.33.6), crashing the backend with - # "grpc service not ready" before it ever loads a model. Pinning - # grpcio-tools to the installed grpcio version keeps the gencode in step with - # the runtime. Backends whose protobuf runtime trails grpcio can set - # GRPCIO_TOOLS_VERSION to the newest generator their runtime accepts. Falls - # back to unpinned when grpcio isn't installed yet. - # See mudler/LocalAI#10718. - local grpcio_tools_spec="grpcio-tools" - local grpcio_version - if [ -n "${GRPCIO_TOOLS_VERSION:-}" ]; then - grpcio_tools_spec="grpcio-tools==${GRPCIO_TOOLS_VERSION}" - else - grpcio_version="$(python -c 'import importlib.metadata as m; print(m.version("grpcio"))' 2>/dev/null || true)" + # The protoc that grpcio-tools bundles stamps a Protobuf "gencode" version + # into backend_pb2.py, and Protobuf refuses to import a stub whose gencode is + # newer than the installed runtime. Left unpinned, grpcio-tools resolves to + # the newest release (1.82.1, gencode 7.35.0) which a backend holding the + # runtime lower (vLLM resolves protobuf to 6.33.6) then cannot import, + # crashing with "grpc service not ready" before it ever loads a model. + # + # Resolve the generator in a THROWAWAY environment so the backend's own venv + # keeps exactly the dependency set its requirements files declared. The + # generator is a build tool and has no business editing the runtime deps: the + # spec below is chosen to fit the venv as it stands, never to change it. + # Falls back to unpinned when neither version can be detected. + # See mudler/LocalAI#10718, #10940. + local protobuf_version grpcio_version protogen_env protogen_python + protobuf_version="$(python -c 'import importlib.metadata as m; print(m.version("protobuf"))' 2>/dev/null || true)" + grpcio_version="$(python -c 'import importlib.metadata as m; print(m.version("grpcio"))' 2>/dev/null || true)" + + # The stubs impose TWO independent constraints on the generator, and both + # have to hold or the backend dies on import: + # + # backend_pb2.py needs protobuf runtime >= gencode + # backend_pb2_grpc.py needs installed grpcio >= grpcio-tools + # + # Bound grpcio-tools from both sides and let the resolver find the newest + # version that satisfies them. The protobuf ceiling makes it back off to an + # older grpcio-tools when the runtime is behind, which is what bounds the + # gencode; the grpcio ceiling keeps the _grpc stub loadable. Pinning only one + # side is what made the earlier attempts fail, in both directions. + local -a protogen_spec=("grpcio-tools") + if [ -n "${grpcio_version}" ]; then + protogen_spec=("grpcio-tools<=${grpcio_version}") fi - if [ "${grpcio_tools_spec}" = "grpcio-tools" ] && [ -n "${grpcio_version}" ]; then - grpcio_tools_spec="grpcio-tools==${grpcio_version}" + if [ -n "${protobuf_version}" ]; then + protogen_spec+=("protobuf<=${protobuf_version}") fi + protogen_env="$(mktemp -d)" + if [ "x${USE_PIP}" == "xtrue" ]; then - pip install "${grpcio_tools_spec}" + python -m venv "${protogen_env}" + "${protogen_env}/bin/pip" install "${protogen_spec[@]}" else - uv pip install "${grpcio_tools_spec}" + uv venv "${protogen_env}" + VIRTUAL_ENV="${protogen_env}" uv pip install "${protogen_spec[@]}" fi + protogen_python="${protogen_env}/bin/python" + pushd "${EDIR}" >/dev/null + # Drop the cached bytecode along with the sources. CPython validates a + # .pyc against the source's mtime *and size*, both of which can be + # unchanged across a regeneration (the gencode triple is the same width + # whether it reads 7.35.0 or 6.33.5), so a stale backend_pb2.pyc can be + # reused in place of the stub we just generated. rm -f backend_pb2.py backend_pb2.pyi backend_pb2_grpc.py - # use the venv python (ensures correct interpreter & sys.path) - python -m grpc_tools.protoc -I../../ -I./ --python_out=. --grpc_python_out=. backend.proto + rm -rf __pycache__ + # Generate with the throwaway toolchain; the output is plain Python and + # carries no dependency on the interpreter that produced it. + "${protogen_python}" -m grpc_tools.protoc -I../../ -I./ --python_out=. --grpc_python_out=. backend.proto + + # Verify with the BACKEND's python, which is the interpreter that has to + # import these at model load. Fail the build rather than ship a backend + # that cannot import its own stubs: otherwise the gencode/runtime + # mismatch only surfaces in a released image, as an opaque + # "grpc service not ready". + # Check BOTH stubs: backend_pb2 catches a gencode ahead of the protobuf + # runtime, backend_pb2_grpc catches generated code ahead of the installed + # grpcio. Checking only the former lets the latter reach CI, or users. + if ! python -c 'import backend_pb2, backend_pb2_grpc' >/dev/null; then + echo "runProtogen: generated stubs are not importable by the backend venv (protobuf ${protobuf_version:-unknown}, grpcio ${grpcio_version:-unknown})" >&2 + exit 1 + fi popd >/dev/null + + rm -rf "${protogen_env}" } diff --git a/backend/python/fish-speech/install.sh b/backend/python/fish-speech/install.sh index 4be2cd2d2..4d03d344c 100644 --- a/backend/python/fish-speech/install.sh +++ b/backend/python/fish-speech/install.sh @@ -60,3 +60,9 @@ if [ "x${USE_PIP}" == "xtrue" ]; then else uv pip install "protobuf>=5.29.0" fi + +# Regenerate the stubs against the protobuf runtime settled on just above. The +# pin exists because transitive deps drag protobuf down; generating before that +# pin is applied is what stamps a gencode the runtime then rejects. +# See mudler/LocalAI#10718. +runProtogen diff --git a/backend/python/llama-cpp-quantization/install.sh b/backend/python/llama-cpp-quantization/install.sh index a9001ffaa..72ecded6e 100755 --- a/backend/python/llama-cpp-quantization/install.sh +++ b/backend/python/llama-cpp-quantization/install.sh @@ -68,3 +68,11 @@ if [ ! -x "${QUANTIZE_BIN}" ] && ! command -v llama-quantize &>/dev/null; then echo "Warning: cmake not found — llama-quantize will not be available. Install cmake or provide llama-quantize on PATH." fi fi + +# The stubs generated at the end of installRequirements were built against the +# protobuf runtime as it stood before the installs above, which resolve their own +# dependencies and can move that runtime. Regenerate now that the dependency set +# is final, so the gencode stamped into backend_pb2.py cannot be newer than the +# runtime that ships. Same failure mode as mudler/LocalAI#10718; runProtogen +# clears the previous stubs first, so this is idempotent. +runProtogen diff --git a/backend/python/trl/install.sh b/backend/python/trl/install.sh index ce0552f87..e37221956 100644 --- a/backend/python/trl/install.sh +++ b/backend/python/trl/install.sh @@ -41,3 +41,11 @@ else uv pip install "gguf>=0.16.0" } fi + +# The stubs generated at the end of installRequirements were built against the +# protobuf runtime as it stood before the installs above, which resolve their own +# dependencies and can move that runtime. Regenerate now that the dependency set +# is final, so the gencode stamped into backend_pb2.py cannot be newer than the +# runtime that ships. Same failure mode as mudler/LocalAI#10718; runProtogen +# clears the previous stubs first, so this is idempotent. +runProtogen diff --git a/backend/python/vibevoice/install.sh b/backend/python/vibevoice/install.sh index 199c46e42..cae3de45a 100755 --- a/backend/python/vibevoice/install.sh +++ b/backend/python/vibevoice/install.sh @@ -38,4 +38,12 @@ if [ ! -d VibeVoice ]; then else uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} . fi -fi \ No newline at end of file +fi + +# The stubs generated at the end of installRequirements were built against the +# protobuf runtime as it stood before the installs above, which resolve their own +# dependencies and can move that runtime. Regenerate now that the dependency set +# is final, so the gencode stamped into backend_pb2.py cannot be newer than the +# runtime that ships. Same failure mode as mudler/LocalAI#10718; runProtogen +# clears the previous stubs first, so this is idempotent. +runProtogen diff --git a/backend/python/vllm-omni/install.sh b/backend/python/vllm-omni/install.sh index b6a0c0dcd..fea6855b4 100755 --- a/backend/python/vllm-omni/install.sh +++ b/backend/python/vllm-omni/install.sh @@ -87,3 +87,9 @@ else fi cd .. + +# vllm-omni lands after installRequirements has already generated the protobuf +# stubs, and it re-resolves the protobuf runtime as it installs. Regenerate now +# that the dependency set is final, so the stubs cannot be newer than the runtime +# that ships. Same failure mode as mudler/LocalAI#10718. +runProtogen diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 8aae26ce1..e8012d12e 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -3,12 +3,6 @@ set -e EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation" -# grpcio 1.82 bundles protoc 7.x, while vLLM's resolved protobuf runtime is -# still 6.33.x. Generate the backend stubs with the newest grpcio-tools release -# that emits protobuf 6.x gencode; the newer grpcio runtime can consume those -# stubs, but protobuf 6.x cannot import 7.x gencode. -export GRPCIO_TOOLS_VERSION="1.78.0" - # Avoid to overcommit the CPU during build # https://github.com/vllm-project/vllm/issues/20079 # https://docs.vllm.ai/en/v0.8.3/serving/env_vars.html @@ -265,3 +259,11 @@ elif [ "x${BUILD_TYPE}" == "x" ] && [ "x${FROM_SOURCE:-}" == "xtrue" ]; then else installRequirements fi + +# installRequirements generates the protobuf stubs at the end of its own run, but +# most branches above install vllm *after* it, and vllm re-resolves the protobuf +# runtime when it lands. Stubs generated against the pre-vllm runtime can end up +# newer than the runtime that finally ships, which is exactly the gencode 7.35.0 / +# runtime 6.33.6 crash in mudler/LocalAI#10718. Regenerate once the dependency set +# is final; runProtogen clears the previous stubs first, so this is idempotent. +runProtogen