mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
d57c1acc85
Minimal base images such as the nvidia-l4t variants ship neither uv nor
pip. When a backend is installed onto a persistent /backends volume there,
`local-ai backends install` downloads the OCI layer fine but the follow-up
install.sh aborts at `uv venv` with "uv: command not found", leaving a
half-built backend with no in-container recovery path.
Add ensureUv(), called from ensureVenv() before any uv-based venv work.
When uv is absent (and not in USE_PIP mode) it bootstraps a private copy
into the backend's own directory (${EDIR}/uv-bin) using the same official
installer the base images use. The directory lives on the persistent
volume, so the download happens only once and survives image upgrades.
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:opus-4.8 [Claude Code]
672 lines
25 KiB
Bash
672 lines
25 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
#
|
|
# use the library by adding the following line to a script:
|
|
# source $(dirname $0)/../common/libbackend.sh
|
|
#
|
|
# If you want to limit what targets a backend can be used on, set the variable LIMIT_TARGETS to a
|
|
# space separated list of valid targets BEFORE sourcing the library, for example to only allow a backend
|
|
# to be used on CUDA and CPU backends:
|
|
#
|
|
# LIMIT_TARGETS="cublas cpu"
|
|
# source $(dirname $0)/../common/libbackend.sh
|
|
#
|
|
# You can use any valid BUILD_TYPE or BUILD_PROFILE, if you need to limit a backend to CUDA 12 only:
|
|
#
|
|
# LIMIT_TARGETS="cublas12"
|
|
# source $(dirname $0)/../common/libbackend.sh
|
|
#
|
|
# You can switch between uv (conda-like) and pip installation methods by setting USE_PIP:
|
|
# USE_PIP=true source $(dirname $0)/../common/libbackend.sh
|
|
#
|
|
# ===================== user-configurable defaults =====================
|
|
PYTHON_VERSION="${PYTHON_VERSION:-3.10}" # e.g. 3.10 / 3.11 / 3.12 / 3.13
|
|
PYTHON_PATCH="${PYTHON_PATCH:-18}" # e.g. 18 -> 3.10.18 ; 13 -> 3.11.13
|
|
PY_STANDALONE_TAG="${PY_STANDALONE_TAG:-20250818}" # release tag date
|
|
# Enable/disable bundling of a portable Python build
|
|
PORTABLE_PYTHON="${PORTABLE_PYTHON:-false}"
|
|
|
|
# If you want to fully pin the filename (including tuned CPU targets), set:
|
|
# PORTABLE_PY_FILENAME="cpython-3.10.18+20250818-x86_64_v3-unknown-linux-gnu-install_only.tar.gz"
|
|
: "${PORTABLE_PY_FILENAME:=}"
|
|
: "${PORTABLE_PY_SHA256:=}" # optional; if set we verify the download
|
|
# =====================================================================
|
|
|
|
# Default to uv if USE_PIP is not set
|
|
if [ "x${USE_PIP:-}" == "x" ]; then
|
|
USE_PIP=false
|
|
fi
|
|
|
|
# ----------------------- helpers -----------------------
|
|
function _is_musl() {
|
|
# detect musl (Alpine, etc)
|
|
if command -v ldd >/dev/null 2>&1; then
|
|
ldd --version 2>&1 | grep -qi musl && return 0
|
|
fi
|
|
# busybox-ish fallback
|
|
if command -v getconf >/dev/null 2>&1; then
|
|
getconf GNU_LIBC_VERSION >/dev/null 2>&1 || return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
function _triple() {
|
|
local os="" arch="" libc="gnu"
|
|
case "$(uname -s)" in
|
|
Linux*) os="unknown-linux" ;;
|
|
Darwin*) os="apple-darwin" ;;
|
|
MINGW*|MSYS*|CYGWIN*) os="pc-windows-msvc" ;; # best-effort for Git Bash
|
|
*) echo "Unsupported OS $(uname -s)"; exit 1;;
|
|
esac
|
|
|
|
case "$(uname -m)" in
|
|
x86_64) arch="x86_64" ;;
|
|
aarch64|arm64) arch="aarch64" ;;
|
|
armv7l) arch="armv7" ;;
|
|
i686|i386) arch="i686" ;;
|
|
ppc64le) arch="ppc64le" ;;
|
|
s390x) arch="s390x" ;;
|
|
riscv64) arch="riscv64" ;;
|
|
*) echo "Unsupported arch $(uname -m)"; exit 1;;
|
|
esac
|
|
|
|
if [[ "$os" == "unknown-linux" ]]; then
|
|
if _is_musl; then
|
|
libc="musl"
|
|
else
|
|
libc="gnu"
|
|
fi
|
|
echo "${arch}-${os}-${libc}"
|
|
else
|
|
echo "${arch}-${os}"
|
|
fi
|
|
}
|
|
|
|
function _portable_dir() {
|
|
echo "${EDIR}/python"
|
|
}
|
|
|
|
function _portable_bin() {
|
|
# python-build-standalone puts python in ./bin
|
|
echo "$(_portable_dir)/bin"
|
|
}
|
|
|
|
function _portable_python() {
|
|
if [ -x "$(_portable_bin)/python3" ]; then
|
|
echo "$(_portable_bin)/python3"
|
|
else
|
|
echo "$(_portable_bin)/python"
|
|
fi
|
|
}
|
|
|
|
|
|
# macOS loader env for the portable CPython
|
|
_macosPortableEnv() {
|
|
if [ "$(uname -s)" = "Darwin" ]; then
|
|
export DYLD_LIBRARY_PATH="$(_portable_dir)/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}"
|
|
export DYLD_FALLBACK_LIBRARY_PATH="$(_portable_dir)/lib${DYLD_FALLBACK_LIBRARY_PATH:+:${DYLD_FALLBACK_LIBRARY_PATH}}"
|
|
fi
|
|
}
|
|
|
|
# Good hygiene on macOS for downloaded/extracted trees
|
|
_unquarantinePortablePython() {
|
|
if [ "$(uname -s)" = "Darwin" ]; then
|
|
command -v xattr >/dev/null 2>&1 && xattr -dr com.apple.quarantine "$(_portable_dir)" || true
|
|
fi
|
|
}
|
|
|
|
# ------------------ ### PORTABLE PYTHON ------------------
|
|
function ensurePortablePython() {
|
|
local pdir="$(_portable_dir)"
|
|
local pbin="$(_portable_bin)"
|
|
local pyexe
|
|
|
|
if [ -x "${pbin}/python3" ] || [ -x "${pbin}/python" ]; then
|
|
_macosPortableEnv
|
|
return 0
|
|
fi
|
|
|
|
mkdir -p "${pdir}"
|
|
local triple="$(_triple)"
|
|
|
|
local full_ver="${PYTHON_VERSION}.${PYTHON_PATCH}"
|
|
local fn=""
|
|
if [ -n "${PORTABLE_PY_FILENAME}" ]; then
|
|
fn="${PORTABLE_PY_FILENAME}"
|
|
else
|
|
# generic asset name: cpython-<full_ver>+<tag>-<triple>-install_only.tar.gz
|
|
fn="cpython-${full_ver}+${PY_STANDALONE_TAG}-${triple}-install_only.tar.gz"
|
|
fi
|
|
|
|
local url="https://github.com/astral-sh/python-build-standalone/releases/download/${PY_STANDALONE_TAG}/${fn}"
|
|
local tmp="${pdir}/${fn}"
|
|
echo "Downloading portable Python: ${fn}"
|
|
# curl with retries; fall back to wget if needed
|
|
if command -v curl >/dev/null 2>&1; then
|
|
curl -L --fail --retry 3 --retry-delay 1 -o "${tmp}" "${url}"
|
|
else
|
|
wget -O "${tmp}" "${url}"
|
|
fi
|
|
|
|
if [ -n "${PORTABLE_PY_SHA256}" ]; then
|
|
echo "${PORTABLE_PY_SHA256} ${tmp}" | sha256sum -c -
|
|
fi
|
|
|
|
echo "Extracting ${fn} -> ${pdir}"
|
|
# always a .tar.gz (we purposely choose install_only)
|
|
tar -xzf "${tmp}" -C "${pdir}"
|
|
rm -f "${tmp}"
|
|
|
|
# Some archives nest a directory; if so, flatten to ${pdir}
|
|
# Find the first dir with a 'bin/python*'
|
|
local inner
|
|
inner="$(find "${pdir}" -type f -path "*/bin/python*" -maxdepth 3 2>/dev/null | head -n1 || true)"
|
|
if [ -n "${inner}" ]; then
|
|
local inner_root
|
|
inner_root="$(dirname "$(dirname "${inner}")")" # .../bin -> root
|
|
if [ "${inner_root}" != "${pdir}" ]; then
|
|
# move contents up one level
|
|
shopt -s dotglob
|
|
mv "${inner_root}/"* "${pdir}/"
|
|
rm -rf "${inner_root}"
|
|
shopt -u dotglob
|
|
fi
|
|
fi
|
|
|
|
_unquarantinePortablePython
|
|
_macosPortableEnv
|
|
# Make sure it's runnable
|
|
pyexe="$(_portable_python)"
|
|
"${pyexe}" -V
|
|
}
|
|
|
|
# init handles the setup of the library
|
|
function init() {
|
|
BACKEND_NAME=${PWD##*/}
|
|
MY_DIR=$(realpath "$(dirname "$0")")
|
|
BUILD_PROFILE=$(getBuildProfile)
|
|
|
|
EDIR=${MY_DIR}
|
|
if [ "x${ENV_DIR:-}" != "x" ]; then
|
|
EDIR=${ENV_DIR}
|
|
fi
|
|
|
|
if [ ! -z "${LIMIT_TARGETS:-}" ]; then
|
|
isValidTarget=$(checkTargets ${LIMIT_TARGETS})
|
|
if [ ${isValidTarget} != true ]; then
|
|
echo "${BACKEND_NAME} can only be used on the following targets: ${LIMIT_TARGETS}"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
echo "Initializing libbackend for ${BACKEND_NAME}"
|
|
}
|
|
|
|
|
|
# getBuildProfile will inspect the system to determine which build profile is appropriate:
|
|
# returns one of the following:
|
|
# - cublas12
|
|
# - cublas13
|
|
# - hipblas
|
|
# - intel
|
|
function getBuildProfile() {
|
|
if [ x"${BUILD_TYPE:-}" == "xcublas" ] || [ x"${BUILD_TYPE:-}" == "xl4t" ]; then
|
|
if [ ! -z "${CUDA_MAJOR_VERSION:-}" ]; then
|
|
echo ${BUILD_TYPE}${CUDA_MAJOR_VERSION}
|
|
else
|
|
echo ${BUILD_TYPE}
|
|
fi
|
|
return 0
|
|
fi
|
|
|
|
if [ -d "/opt/intel" ]; then
|
|
echo "intel"
|
|
return 0
|
|
fi
|
|
|
|
if [ -n "${BUILD_TYPE:-}" ]; then
|
|
echo ${BUILD_TYPE}
|
|
return 0
|
|
fi
|
|
|
|
echo "cpu"
|
|
}
|
|
|
|
|
|
# Make the venv relocatable:
|
|
# - rewrite venv/bin/python{,3} to relative symlinks into $(_portable_dir)
|
|
# - normalize entrypoint shebangs to /usr/bin/env python3
|
|
# - optionally update pyvenv.cfg to point to the portable Python directory (only at runtime)
|
|
# Usage: _makeVenvPortable [--update-pyvenv-cfg]
|
|
_makeVenvPortable() {
|
|
local update_pyvenv_cfg=false
|
|
if [ "${1:-}" = "--update-pyvenv-cfg" ]; then
|
|
update_pyvenv_cfg=true
|
|
fi
|
|
|
|
local venv_dir="${EDIR}/venv"
|
|
local vbin="${venv_dir}/bin"
|
|
|
|
[ -d "${vbin}" ] || return 0
|
|
|
|
# 1) Replace python symlinks with relative ones to ../../python/bin/python3
|
|
# (venv/bin -> venv -> EDIR -> python/bin)
|
|
local rel_py='../../python/bin/python3'
|
|
|
|
for name in python3 python; do
|
|
if [ -e "${vbin}/${name}" ] || [ -L "${vbin}/${name}" ]; then
|
|
rm -f "${vbin}/${name}"
|
|
fi
|
|
done
|
|
ln -s "${rel_py}" "${vbin}/python3"
|
|
ln -s "python3" "${vbin}/python"
|
|
|
|
# 2) Update pyvenv.cfg to point to the portable Python directory (only at runtime)
|
|
# Use absolute path resolved at runtime so it works when the venv is copied
|
|
if [ "$update_pyvenv_cfg" = "true" ]; then
|
|
local pyvenv_cfg="${venv_dir}/pyvenv.cfg"
|
|
if [ -f "${pyvenv_cfg}" ]; then
|
|
local portable_dir="$(_portable_dir)"
|
|
# Resolve to absolute path - this ensures it works when the backend is copied
|
|
# Only resolve if the directory exists (it should if ensurePortablePython was called)
|
|
if [ -d "${portable_dir}" ]; then
|
|
portable_dir="$(cd "${portable_dir}" && pwd)"
|
|
else
|
|
# Fallback to relative path if directory doesn't exist yet
|
|
portable_dir="../python"
|
|
fi
|
|
local sed_i=(sed -i)
|
|
# macOS/BSD sed needs a backup suffix; GNU sed doesn't. Make it portable:
|
|
if sed --version >/dev/null 2>&1; then
|
|
sed_i=(sed -i)
|
|
else
|
|
sed_i=(sed -i '')
|
|
fi
|
|
# Update the home field in pyvenv.cfg
|
|
# Handle both absolute paths (starting with /) and relative paths
|
|
if grep -q "^home = " "${pyvenv_cfg}"; then
|
|
"${sed_i[@]}" "s|^home = .*|home = ${portable_dir}|" "${pyvenv_cfg}"
|
|
else
|
|
# If home field doesn't exist, add it
|
|
echo "home = ${portable_dir}" >> "${pyvenv_cfg}"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# 3) Rewrite shebangs of entry points to use env, so the venv is relocatable
|
|
# Only touch text files that start with #! and reference the current venv.
|
|
local ve_abs="${vbin}/python"
|
|
local sed_i=(sed -i)
|
|
# macOS/BSD sed needs a backup suffix; GNU sed doesn't. Make it portable:
|
|
if sed --version >/dev/null 2>&1; then
|
|
sed_i=(sed -i)
|
|
else
|
|
sed_i=(sed -i '')
|
|
fi
|
|
|
|
for f in "${vbin}"/*; do
|
|
[ -f "$f" ] || continue
|
|
# Fast path: check first two bytes (#!)
|
|
head -c2 "$f" 2>/dev/null | grep -q '^#!' || continue
|
|
# Only rewrite if the shebang mentions the (absolute) venv python
|
|
if head -n1 "$f" | grep -Fq "${ve_abs}"; then
|
|
"${sed_i[@]}" '1s|^#!.*$|#!/usr/bin/env python3|' "$f"
|
|
chmod +x "$f" 2>/dev/null || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
|
|
# Apply the venv to the current process: VIRTUAL_ENV, PATH, PYTHONHOME hygiene.
|
|
# Equivalent to the runtime portion of `source bin/activate`, but computed from
|
|
# $EDIR (resolved at runtime via realpath) instead of the path baked into
|
|
# bin/activate at venv-create time. `uv venv` (and `python -m venv`) both bake
|
|
# the create-time absolute path in, so sourcing activate on a relocated venv —
|
|
# e.g. one built at /vllm/venv inside a Docker stage and unpacked under
|
|
# /backends/cuda13-vllm-development/venv at runtime — silently prepends a
|
|
# stale, non-existent path to $PATH. Doing the setup ourselves sidesteps that;
|
|
# this is the same approach `uv run` takes internally.
|
|
_activateVenv() {
|
|
export VIRTUAL_ENV="${EDIR}/venv"
|
|
export PATH="${EDIR}/venv/bin:${PATH}"
|
|
unset PYTHONHOME
|
|
}
|
|
|
|
# ensureUv makes sure the `uv` package manager is on PATH before we try to build
|
|
# or populate a venv with it.
|
|
#
|
|
# Minimal base images (notably the nvidia-l4t variants) ship neither uv nor pip.
|
|
# A backend installed onto a persistent /backends volume there can therefore
|
|
# never build its venv: `uv venv` aborts with "uv: command not found" and
|
|
# install.sh leaves a half-installed backend, with no way to recover from inside
|
|
# the container. See mudler/LocalAI#10720.
|
|
#
|
|
# When uv is missing (and we are not in USE_PIP mode) bootstrap a private copy
|
|
# into the backend directory itself, which lives on the same persistent volume,
|
|
# so it survives image upgrades and is only downloaded once.
|
|
function ensureUv() {
|
|
# pip mode never invokes uv.
|
|
if [ "x${USE_PIP}" == "xtrue" ]; then
|
|
return 0
|
|
fi
|
|
if command -v uv >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
local uv_dir="${EDIR}/uv-bin"
|
|
if [ -x "${uv_dir}/uv" ]; then
|
|
export PATH="${uv_dir}:${PATH}"
|
|
return 0
|
|
fi
|
|
echo "uv not found; bootstrapping a private copy into ${uv_dir}"
|
|
mkdir -p "${uv_dir}"
|
|
# The official installer picks the right binary for the current platform and
|
|
# honors UV_INSTALL_DIR. This is the same mechanism the base images use to
|
|
# install uv (see backend/Dockerfile.python). INSTALLER_NO_MODIFY_PATH keeps
|
|
# it from editing shell profiles we don't control.
|
|
if command -v curl >/dev/null 2>&1; then
|
|
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="${uv_dir}" INSTALLER_NO_MODIFY_PATH=1 sh
|
|
elif command -v wget >/dev/null 2>&1; then
|
|
wget -qO- https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="${uv_dir}" INSTALLER_NO_MODIFY_PATH=1 sh
|
|
else
|
|
echo "ERROR: uv is missing and neither curl nor wget is available to bootstrap it" >&2
|
|
return 1
|
|
fi
|
|
export PATH="${uv_dir}:${PATH}"
|
|
command -v uv >/dev/null 2>&1
|
|
}
|
|
|
|
# ensureVenv makes sure that the venv for the backend both exists, and is activated.
|
|
#
|
|
# This function is idempotent, so you can call it as many times as you want and it will
|
|
# always result in an activated virtual environment
|
|
function ensureVenv() {
|
|
local interpreter=""
|
|
|
|
# Make sure uv is available before any uv-based venv work below.
|
|
ensureUv
|
|
|
|
if [ "x${PORTABLE_PYTHON}" == "xtrue" ] || [ -e "$(_portable_python)" ]; then
|
|
echo "Using portable Python"
|
|
ensurePortablePython
|
|
interpreter="$(_portable_python)"
|
|
else
|
|
# Prefer system python${PYTHON_VERSION}, else python3, else fall back to bundled
|
|
if command -v python${PYTHON_VERSION} >/dev/null 2>&1; then
|
|
interpreter="python${PYTHON_VERSION}"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
interpreter="python3"
|
|
else
|
|
echo "No suitable system Python found, bootstrapping portable build..."
|
|
ensurePortablePython
|
|
interpreter="$(_portable_python)"
|
|
fi
|
|
fi
|
|
|
|
if [ ! -d "${EDIR}/venv" ]; then
|
|
if [ "x${USE_PIP}" == "xtrue" ]; then
|
|
# --copies is only needed when we will later relocate the venv via
|
|
# _makeVenvPortable (PORTABLE_PYTHON=true). Some Python builds —
|
|
# notably macOS system Python — refuse to create a venv with
|
|
# --copies because the build doesn't support it. Fall back to
|
|
# symlinks in that case.
|
|
local venv_args=""
|
|
if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then
|
|
venv_args="--copies"
|
|
fi
|
|
"${interpreter}" -m venv ${venv_args} "${EDIR}/venv"
|
|
_activateVenv
|
|
"${interpreter}" -m pip install --upgrade pip
|
|
else
|
|
if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then
|
|
uv venv --python "${interpreter}" "${EDIR}/venv"
|
|
else
|
|
uv venv --python "${PYTHON_VERSION}" "${EDIR}/venv"
|
|
fi
|
|
fi
|
|
if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then
|
|
# During install, only update symlinks and shebangs, not pyvenv.cfg
|
|
_makeVenvPortable
|
|
fi
|
|
fi
|
|
|
|
# We call it here to make sure that when we source a venv we can still use python as expected
|
|
if [ -x "$(_portable_python)" ]; then
|
|
_macosPortableEnv
|
|
fi
|
|
|
|
if [ "x${VIRTUAL_ENV:-}" != "x${EDIR}/venv" ]; then
|
|
_activateVenv
|
|
fi
|
|
}
|
|
|
|
|
|
function runProtogen() {
|
|
ensureVenv
|
|
|
|
# The protoc that grpcio-tools bundles stamps a Protobuf "gencode" version
|
|
# into backend_pb2.py, and Protobuf refuses to import a stub whose gencode is
|
|
# newer than the installed runtime. Left unpinned, grpcio-tools resolves to
|
|
# the newest release (1.82.1, gencode 7.35.0) which a backend holding the
|
|
# runtime lower (vLLM resolves protobuf to 6.33.6) then cannot import,
|
|
# crashing with "grpc service not ready" before it ever loads a model.
|
|
#
|
|
# Resolve the generator in a THROWAWAY environment so the backend's own venv
|
|
# keeps exactly the dependency set its requirements files declared. The
|
|
# generator is a build tool and has no business editing the runtime deps: the
|
|
# spec below is chosen to fit the venv as it stands, never to change it.
|
|
# Falls back to unpinned when neither version can be detected.
|
|
# See mudler/LocalAI#10718, #10940.
|
|
local protobuf_version grpcio_version protogen_env protogen_python
|
|
protobuf_version="$(python -c 'import importlib.metadata as m; print(m.version("protobuf"))' 2>/dev/null || true)"
|
|
grpcio_version="$(python -c 'import importlib.metadata as m; print(m.version("grpcio"))' 2>/dev/null || true)"
|
|
|
|
# The stubs impose TWO independent constraints on the generator, and both
|
|
# have to hold or the backend dies on import:
|
|
#
|
|
# backend_pb2.py needs protobuf runtime >= gencode
|
|
# backend_pb2_grpc.py needs installed grpcio >= grpcio-tools
|
|
#
|
|
# Bound grpcio-tools from both sides and let the resolver find the newest
|
|
# version that satisfies them. The protobuf ceiling makes it back off to an
|
|
# older grpcio-tools when the runtime is behind, which is what bounds the
|
|
# gencode; the grpcio ceiling keeps the _grpc stub loadable. Pinning only one
|
|
# side is what made the earlier attempts fail, in both directions.
|
|
local -a protogen_spec=("grpcio-tools")
|
|
if [ -n "${grpcio_version}" ]; then
|
|
protogen_spec=("grpcio-tools<=${grpcio_version}")
|
|
fi
|
|
if [ -n "${protobuf_version}" ]; then
|
|
protogen_spec+=("protobuf<=${protobuf_version}")
|
|
fi
|
|
|
|
protogen_env="$(mktemp -d)"
|
|
|
|
if [ "x${USE_PIP}" == "xtrue" ]; then
|
|
python -m venv "${protogen_env}"
|
|
"${protogen_env}/bin/pip" install "${protogen_spec[@]}"
|
|
else
|
|
uv venv "${protogen_env}"
|
|
VIRTUAL_ENV="${protogen_env}" uv pip install "${protogen_spec[@]}"
|
|
fi
|
|
protogen_python="${protogen_env}/bin/python"
|
|
|
|
pushd "${EDIR}" >/dev/null
|
|
# Drop the cached bytecode along with the sources. CPython validates a
|
|
# .pyc against the source's mtime *and size*, both of which can be
|
|
# unchanged across a regeneration (the gencode triple is the same width
|
|
# whether it reads 7.35.0 or 6.33.5), so a stale backend_pb2.pyc can be
|
|
# reused in place of the stub we just generated.
|
|
rm -f backend_pb2.py backend_pb2.pyi backend_pb2_grpc.py
|
|
rm -rf __pycache__
|
|
# Generate with the throwaway toolchain; the output is plain Python and
|
|
# carries no dependency on the interpreter that produced it.
|
|
"${protogen_python}" -m grpc_tools.protoc -I../../ -I./ --python_out=. --grpc_python_out=. backend.proto
|
|
|
|
# Verify with the BACKEND's python, which is the interpreter that has to
|
|
# import these at model load. Fail the build rather than ship a backend
|
|
# that cannot import its own stubs: otherwise the gencode/runtime
|
|
# mismatch only surfaces in a released image, as an opaque
|
|
# "grpc service not ready".
|
|
# Check BOTH stubs: backend_pb2 catches a gencode ahead of the protobuf
|
|
# runtime, backend_pb2_grpc catches generated code ahead of the installed
|
|
# grpcio. Checking only the former lets the latter reach CI, or users.
|
|
if ! python -c 'import backend_pb2, backend_pb2_grpc' >/dev/null; then
|
|
echo "runProtogen: generated stubs are not importable by the backend venv (protobuf ${protobuf_version:-unknown}, grpcio ${grpcio_version:-unknown})" >&2
|
|
exit 1
|
|
fi
|
|
popd >/dev/null
|
|
|
|
rm -rf "${protogen_env}"
|
|
}
|
|
|
|
|
|
# installRequirements looks for several requirements files and if they exist runs the install for them in order
|
|
#
|
|
# - requirements-install.txt
|
|
# - requirements.txt
|
|
# - requirements-${BUILD_TYPE}.txt
|
|
# - requirements-${BUILD_PROFILE}.txt
|
|
#
|
|
# BUILD_PROFILE is a more specific version of BUILD_TYPE, ex: cuda-12 or cuda-13
|
|
# it can also include some options that we do not have BUILD_TYPES for, ex: intel
|
|
#
|
|
# NOTE: for BUILD_PROFILE==intel, this function does NOT automatically use the Intel python package index.
|
|
# you may want to add the following line to a requirements-intel.txt if you use one:
|
|
#
|
|
# --index-url https://download.pytorch.org/whl/xpu
|
|
#
|
|
# If you need to add extra flags into the pip install command you can do so by setting the variable EXTRA_PIP_INSTALL_FLAGS
|
|
# before calling installRequirements. For example:
|
|
#
|
|
# source $(dirname $0)/../common/libbackend.sh
|
|
# EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation"
|
|
# installRequirements
|
|
function installRequirements() {
|
|
ensureVenv
|
|
declare -a requirementFiles=(
|
|
"${EDIR}/requirements-install.txt"
|
|
"${EDIR}/requirements.txt"
|
|
"${EDIR}/requirements-${BUILD_TYPE:-}.txt"
|
|
)
|
|
|
|
if [ "x${BUILD_TYPE:-}" != "x${BUILD_PROFILE}" ]; then
|
|
requirementFiles+=("${EDIR}/requirements-${BUILD_PROFILE}.txt")
|
|
fi
|
|
if [ "x${BUILD_TYPE:-}" == "x" ]; then
|
|
requirementFiles+=("${EDIR}/requirements-cpu.txt")
|
|
fi
|
|
requirementFiles+=("${EDIR}/requirements-after.txt")
|
|
if [ "x${BUILD_TYPE:-}" != "x${BUILD_PROFILE}" ]; then
|
|
requirementFiles+=("${EDIR}/requirements-${BUILD_PROFILE}-after.txt")
|
|
fi
|
|
|
|
# This is needed to build wheels that e.g. depends on Python.h
|
|
if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then
|
|
export C_INCLUDE_PATH="${C_INCLUDE_PATH:-}:$(_portable_dir)/include/python${PYTHON_VERSION}"
|
|
fi
|
|
|
|
for reqFile in ${requirementFiles[@]}; do
|
|
if [ -f "${reqFile}" ]; then
|
|
echo "starting requirements install for ${reqFile}"
|
|
if [ "x${USE_PIP}" == "xtrue" ]; then
|
|
pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}"
|
|
else
|
|
uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}"
|
|
fi
|
|
echo "finished requirements install for ${reqFile}"
|
|
fi
|
|
done
|
|
|
|
runProtogen
|
|
}
|
|
|
|
# startBackend discovers and runs the backend GRPC server
|
|
#
|
|
# You can specify a specific backend file to execute by setting BACKEND_FILE before calling startBackend.
|
|
# example:
|
|
#
|
|
# source ../common/libbackend.sh
|
|
# BACKEND_FILE="${MY_DIR}/source/backend.py"
|
|
# startBackend $@
|
|
#
|
|
# valid filenames for autodiscovered backend servers are:
|
|
# - server.py
|
|
# - backend.py
|
|
# - ${BACKEND_NAME}.py
|
|
function startBackend() {
|
|
ensureVenv
|
|
# Update pyvenv.cfg before running to ensure paths are correct for current location
|
|
# This is critical when the backend position is dynamic (e.g., copied from container)
|
|
if [ "x${PORTABLE_PYTHON}" == "xtrue" ] || [ -x "$(_portable_python)" ]; then
|
|
_makeVenvPortable --update-pyvenv-cfg
|
|
fi
|
|
|
|
# Set up GPU library paths if a lib directory exists
|
|
# This allows backends to include their own GPU libraries (CUDA, ROCm, etc.)
|
|
if [ -d "${EDIR}/lib" ]; then
|
|
export LD_LIBRARY_PATH="${EDIR}/lib:${LD_LIBRARY_PATH:-}"
|
|
echo "Added ${EDIR}/lib to LD_LIBRARY_PATH for GPU libraries"
|
|
fi
|
|
|
|
if [ ! -z "${BACKEND_FILE:-}" ]; then
|
|
exec "${EDIR}/venv/bin/python" "${BACKEND_FILE}" "$@"
|
|
elif [ -e "${MY_DIR}/server.py" ]; then
|
|
exec "${EDIR}/venv/bin/python" "${MY_DIR}/server.py" "$@"
|
|
elif [ -e "${MY_DIR}/backend.py" ]; then
|
|
exec "${EDIR}/venv/bin/python" "${MY_DIR}/backend.py" "$@"
|
|
elif [ -e "${MY_DIR}/${BACKEND_NAME}.py" ]; then
|
|
exec "${EDIR}/venv/bin/python" "${MY_DIR}/${BACKEND_NAME}.py" "$@"
|
|
fi
|
|
}
|
|
|
|
|
|
# runUnittests discovers and runs python unittests
|
|
#
|
|
# You can specify a specific test file to use by setting TEST_FILE before calling runUnittests.
|
|
# example:
|
|
#
|
|
# source ../common/libbackend.sh
|
|
# TEST_FILE="${MY_DIR}/source/test.py"
|
|
# runUnittests $@
|
|
#
|
|
# be default a file named test.py in the backends directory will be used
|
|
function runUnittests() {
|
|
ensureVenv
|
|
if [ ! -z "${TEST_FILE:-}" ]; then
|
|
testDir=$(dirname "$(realpath "${TEST_FILE}")")
|
|
testFile=$(basename "${TEST_FILE}")
|
|
pushd "${testDir}" >/dev/null
|
|
python -m unittest "${testFile}"
|
|
popd >/dev/null
|
|
elif [ -f "${MY_DIR}/test.py" ]; then
|
|
pushd "${MY_DIR}" >/dev/null
|
|
python -m unittest test.py
|
|
popd >/dev/null
|
|
else
|
|
echo "no tests defined for ${BACKEND_NAME}"
|
|
fi
|
|
}
|
|
|
|
|
|
##################################################################################
|
|
# Below here are helper functions not intended to be used outside of the library #
|
|
##################################################################################
|
|
|
|
# checkTargets determines if the current BUILD_TYPE or BUILD_PROFILE is in a list of valid targets
|
|
function checkTargets() {
|
|
targets=$@
|
|
declare -a targets=($targets)
|
|
for target in ${targets[@]}; do
|
|
if [ "x${BUILD_TYPE:-}" == "x${target}" ]; then
|
|
echo true; return 0
|
|
fi
|
|
if [ "x${BUILD_PROFILE}" == "x${target}" ]; then
|
|
echo true; return 0
|
|
fi
|
|
done
|
|
echo false
|
|
}
|
|
|
|
init
|