diff --git a/backend/python/common/libbackend.sh b/backend/python/common/libbackend.sh index 14172decf..8dc52e4ff 100644 --- a/backend/python/common/libbackend.sh +++ b/backend/python/common/libbackend.sh @@ -563,6 +563,18 @@ function startBackend() { echo "Added ${EDIR}/lib to LD_LIBRARY_PATH for GPU libraries" fi + # XPU wheels carry a matching SYCL/Unified Runtime set. Prefer it over + # packaged oneAPI or host libraries, which may expose an older loader ABI. + if [ "$(uname -s)" = "Linux" ]; then + local sycl_runtime + for sycl_runtime in "${EDIR}/venv/lib"/libsycl.so*; do + if [ -f "${sycl_runtime}" ]; then + export LD_LIBRARY_PATH="${EDIR}/venv/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + break + fi + done + fi + if [ ! -z "${BACKEND_FILE:-}" ]; then exec "${EDIR}/venv/bin/python" "${BACKEND_FILE}" "$@" elif [ -e "${MY_DIR}/server.py" ]; then diff --git a/docs/content/getting-started/troubleshooting.md b/docs/content/getting-started/troubleshooting.md index 3e925125b..3527ac9d9 100644 --- a/docs/content/getting-started/troubleshooting.md +++ b/docs/content/getting-started/troubleshooting.md @@ -178,6 +178,31 @@ Use container images with `gpu-intel` in the tag. **Known issue:** SYCL hangs wh mmap: false ``` +If the Intel diffusers backend exits while importing PyTorch with +`libsycl.so.9: undefined symbol: urDeviceWaitExp, version LIBUR_LOADER_0.12`, +the SYCL library has loaded an incompatible Unified Runtime loader. This happens +before the gRPC server starts; later connection-refused errors are a consequence +of that import failure. + +Update the separately installed Intel **diffusers backend** as well as the +`gpu-intel` container image. Updating the parent image alone does not replace a +backend retained in your backends directory. On Linux, the Python backend launcher +prefers `venv/lib` when it contains `libsycl.so*`, ahead of the backend's packaged +`lib` directory and inherited `LD_LIBRARY_PATH`. The venv still needs a compatible +SYCL library, Unified Runtime loader and adapters; changing search order cannot +repair an incomplete runtime installation. + +If the failure persists, include the container image digest, installed backend +name and OCI digest, full import traceback, launch command, and any library mounts +in your issue. From inside the affected container, record the inherited +`LD_LIBRARY_PATH`, the installed backend path, and the `libsycl.so*`, +`libur_loader.so*` and `libur_adapter_*` files in both `venv/lib` and `lib`. +Use `readelf --dyn-syms --wide` and `readelf --version-info` on the SYCL library and +loaders to compare their symbols and versions. Running the installed backend's +`run.sh` with `LD_DEBUG=libs,versions` captures which loader is actually selected +under its startup environment. An `import torch` check using the venv Python and +the same library search path can isolate this failure without generating an image. + **Overriding backend auto-detection:** If LocalAI picks the wrong GPU backend, override it: diff --git a/scripts/build/python-backend-intel-runtime_test.sh b/scripts/build/python-backend-intel-runtime_test.sh new file mode 100755 index 000000000..43d7be89e --- /dev/null +++ b/scripts/build/python-backend-intel-runtime_test.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# Exercise the real diffusers launcher with versioned ELF libraries, no torch, +# downloads, venv creation or GPU required. +set -euo pipefail + +if [ "$(uname -s)" != "Linux" ]; then + echo "SKIP: Intel Python runtime regression requires Linux ELF loading" + exit 0 +fi +for tool in gcc ldd python3; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "SKIP: $tool not available" + exit 0 + fi +done +if ! ldd --version 2>&1 | grep -qi 'glibc\|GNU libc'; then + echo "SKIP: versioned ELF regression requires glibc" + exit 0 +fi + +REPO_ROOT=$(dirname "$(dirname "$(dirname "$(realpath "$0")")")") +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +export TEST_SYSTEM_PYTHON +TEST_SYSTEM_PYTHON=$(command -v python3) +unset ENV_DIR BACKEND_FILE VIRTUAL_ENV PYTHONHOME LD_LIBRARY_PATH +export PORTABLE_PYTHON=false BUILD_TYPE=cpu + +# Fail immediately if startup attempts to provision anything. +mkdir -p "$WORK/bin" "$WORK/good" "$WORK/bad" "$WORK/host" "$WORK/cwd" +for tool in uv pip pip3 curl wget; do + printf '#!/bin/bash\necho "Unexpected provisioning command" >&2\nexit 99\n' > "$WORK/bin/$tool" + chmod +x "$WORK/bin/$tool" +done +export PATH="$WORK/bin:$PATH" + +cat > "$WORK/loader.c" <<'C' +int urFixtureOrigin(void) { return ORIGIN; } +#ifdef MATCHING +int urDeviceWaitExp(void) { return 42; } +#else +int urOldFunction(void) { return 0; } +#endif +C +cat > "$WORK/loader.map" <<'MAP' +LIBUR_LOADER_0.12 { global: ur*; local: *; }; +MAP +cat > "$WORK/sycl.c" <<'C' +int urDeviceWaitExp(void); +int syclFixture(void) { return urDeviceWaitExp(); } +C +echo 'int adapterFixture(void) { return ORIGIN; }' > "$WORK/adapter.c" + +# All loaders have the same SONAME and version namespace, but only the venv +# copy exports the required versioned symbol (the reported failure class). +for kind in good bad host; do + flags=() + case "$kind" in + good) flags=(-DMATCHING -DORIGIN=1) ;; + bad) flags=(-DORIGIN=2) ;; + host) flags=(-DORIGIN=3) ;; + esac + gcc -shared -fPIC "${flags[@]}" "$WORK/loader.c" \ + -Wl,-soname,libur_loader.so.0 -Wl,--version-script,"$WORK/loader.map" \ + -o "$WORK/$kind/libur_loader.so.0" + gcc -shared -fPIC "${flags[@]}" "$WORK/adapter.c" \ + -o "$WORK/$kind/libur_adapter_level_zero.so.0" +done +# The ELF loader expands $ORIGIN relative to the loaded library. +# shellcheck disable=SC2016 +gcc -shared -fPIC "$WORK/sycl.c" -L"$WORK/good" -l:libur_loader.so.0 \ + -Wl,-soname,libsycl.so.9 -Wl,-rpath,'$ORIGIN' \ + -o "$WORK/good/libsycl.so.9" + +cat > "$WORK/backend.py" <<'PY' +import ctypes +import os +import sys + +assert sys.argv[1:] == ["--fixture"], sys.argv +assert os.environ.get("LD_LIBRARY_PATH") == os.environ.get("EXPECTED_PATH"), ( + os.environ.get("LD_LIBRARY_PATH"), os.environ.get("EXPECTED_PATH") +) +if os.environ.get("LOAD_SYCL") == "1": + sycl = ctypes.CDLL(os.path.join(os.environ["VIRTUAL_ENV"], "lib", "libsycl.so.9")) + assert sycl.syclFixture() == 42 +if "EXPECTED_ORIGIN" in os.environ: + expected = int(os.environ["EXPECTED_ORIGIN"]) + loader = ctypes.CDLL("libur_loader.so.0") + assert loader.urFixtureOrigin() == expected + adapter = ctypes.CDLL("libur_adapter_level_zero.so.0") + assert adapter.adapterFixture() == expected +print("fixture passed") +PY + +make_backend() { + local dir="$1" env_dir="${2:-$1}" + mkdir -p "$dir/common" "$env_dir/venv/bin" "$env_dir/venv/lib" "$env_dir/lib" + cp "$REPO_ROOT/backend/python/diffusers/run.sh" "$dir/run.sh" + cp "$REPO_ROOT/backend/python/common/libbackend.sh" "$dir/common/libbackend.sh" + cp "$WORK/backend.py" "$dir/backend.py" + cat > "$env_dir/venv/bin/python" <<'SH' +#!/bin/bash +exec "$TEST_SYSTEM_PYTHON" "$@" +SH + chmod +x "$env_dir/venv/bin/python" + cp "$WORK/good/"* "$env_dir/venv/lib/" + cp "$WORK/bad/"* "$env_dir/lib/" +} + +launch() { + (cd "$WORK/cwd"; bash "$1/run.sh" --fixture) +} + +expect_failure() { + if "$@" > "$WORK/failure.log" 2>&1; then + echo "FAIL: incompatible loader unexpectedly succeeded" + exit 1 + fi + # A path assertion or launcher error must not count as the expected failure. + if ! grep -q 'undefined symbol: urDeviceWaitExp, version LIBUR_LOADER_0.12' "$WORK/failure.log"; then + cat "$WORK/failure.log" + exit 1 + fi +} + +backend="$WORK/built/diffusers" +make_backend "$backend" +mv "$WORK/built" "$WORK/relocated" +backend="$WORK/relocated/diffusers" +export LOAD_SYCL=1 EXPECTED_ORIGIN=1 + +# Reproduce the old ordering using the same venv Python shim and backend file. +export LD_LIBRARY_PATH="$backend/lib:$WORK/host" +export EXPECTED_PATH="$LD_LIBRARY_PATH" VIRTUAL_ENV="$backend/venv" +expect_failure "$backend/venv/bin/python" "$backend/backend.py" --fixture +unset VIRTUAL_ENV + +# Both inherited and packaged conflicts lose to the venv runtime. +export LD_LIBRARY_PATH="$WORK/host" EXPECTED_PATH="$backend/venv/lib:$backend/lib:$WORK/host" +launch "$backend" + +# Every startBackend exec branch must receive the same search path. +export BACKEND_FILE="$backend/backend.py" +launch "$backend" +unset BACKEND_FILE +mv "$backend/backend.py" "$backend/server.py" +launch "$backend" +# BACKEND_NAME currently derives from the caller's working directory. +mv "$backend/server.py" "$backend/cwd.py" +launch "$backend" +mv "$backend/cwd.py" "$backend/backend.py" + +unset LD_LIBRARY_PATH +export EXPECTED_PATH="$backend/venv/lib:$backend/lib:" +launch "$backend" + +# No packaged directory: the venv still takes precedence, with no empty entry +# introduced when the inherited path is unset. +mv "$backend/lib" "$backend/packaged" +export EXPECTED_PATH="$backend/venv/lib" +launch "$backend" +export LD_LIBRARY_PATH="$WORK/host" EXPECTED_PATH="$backend/venv/lib:$WORK/host" +launch "$backend" +mv "$backend/packaged" "$backend/lib" + +# The launcher accepts a relative path when its parent directory has spaces; +# its existing unquoted $0 handling does not accept absolute paths with spaces. +spaced="$WORK/relocated with spaces" +make_backend "$spaced/diffusers" +mkdir -p "$spaced/cwd" +export EXPECTED_PATH="$spaced/diffusers/venv/lib:$spaced/diffusers/lib:$WORK/host" +(cd "$spaced/cwd"; bash ../diffusers/run.sh --fixture) + +separate="$WORK/separate environment" +make_backend "$WORK/external" "$separate" +export ENV_DIR="$separate" EXPECTED_PATH="$separate/venv/lib:$separate/lib:$WORK/host" +launch "$WORK/external" +unset ENV_DIR + +# Missing and incompatible venv loaders must still fail, even though the venv +# directory and SYCL library exist. Presence is not an ABI compatibility check. +export EXPECTED_PATH="$backend/venv/lib:$backend/lib:$WORK/host" +rm "$backend/venv/lib/libur_loader.so.0" +expect_failure launch "$backend" +cp "$WORK/bad/libur_loader.so.0" "$backend/venv/lib/" +expect_failure launch "$backend" + +# Without a SYCL library, unrelated CPU/CUDA files (or even a directory named +# libsycl.so) must not change the existing library search order. +rm "$backend/venv/lib/libsycl.so.9" +mkdir "$backend/venv/lib/libsycl.so" +touch "$backend/venv/lib/libcublas.so.12" "$backend/venv/lib/libcpu.so" +export LOAD_SYCL=0 EXPECTED_ORIGIN=2 EXPECTED_PATH="$backend/lib:$WORK/host" +launch "$backend" +unset LD_LIBRARY_PATH +export EXPECTED_PATH="$backend/lib:" +launch "$backend" +rm -rf "${backend:?}/lib" +export LD_LIBRARY_PATH="$WORK/host" EXPECTED_PATH="$WORK/host" EXPECTED_ORIGIN=3 +launch "$backend" +unset LD_LIBRARY_PATH EXPECTED_PATH EXPECTED_ORIGIN +launch "$backend" + +echo "PASS: Intel Python runtime precedence, relocation, fallback and import errors"