fix(ci): unbreak the e2e build and the darwin vllm-metal pin (#11849)

Two independent breakages on master make every open pull request red,
for reasons unrelated to the changes under review.

The e2e backend suite stopped compiling. Reply.message is `bytes` in
backend.proto, so res.GetMessage() returns []byte, and strings.ToUpper
wants a string. Every other call site in the file already converts.
tests/e2e-backends sits behind a build tag, so `go build ./...` never
compiled it and the breakage reached master unnoticed.

The darwin vllm build stopped resolving. Upstream vllm-metal deleted
its old dev tags and re-versioned to track the vLLM release it targets,
so the pinned wheel 404s. The coupled vLLM release also moved out of
upstream's install.sh into .github/vllm-release-tag.commit, and the
wheel's platform tag moved from macosx_11_0 to macosx_15_0.

Read the wheel name from the release's own asset listing rather than
composing it from a hardcoded platform segment, so a platform-tag
change cannot silently 404 again, and resolve the vLLM version from
the new metadata file with a fallback to the legacy installer. The
bump script and the extractor learn the same two-source lookup, so the
next nightly run converges on the pin checked in here instead of
reintroducing the break.

Assisted-by: Claude:claude-opus-5

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

.github/bump_vllm_metal.sh

@@ -3,9 +3,9 @@
 # darwin (Apple Silicon) install path. The macOS/Metal build
 # (backend/python/vllm/install.sh, Darwin branch) installs vllm-metal, which is
 # version-locked to a specific vLLM source release. install.sh derives that vLLM
-# version at build time from vllm-metal's own installer at the pinned
-# tag, so there is only ONE value to bump here -- mirroring bump_vllm_wheel.sh,
-# which bumps the Linux cu130 wheel pin.
+# version, and the wheel asset name, at build time from the pinned tag, so there
+# is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, which bumps the
+# Linux cu130 wheel pin.
 #
 # This deliberately tracks vllm-project/vllm-metal, NOT vllm-project/vllm: the
 # darwin build can only use the exact vLLM version vllm-metal supports, so it may
@@ -23,15 +23,20 @@ if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then
     exit 1
 fi
 
-# vllm-metal ships frequent dev releases, all flagged as non-prerelease, so
-# /releases/latest returns the newest one (with its cp312 wheel asset).
+# vllm-metal ships frequent .dev releases, flagged as prereleases, alongside the
+# stable ones. /releases/latest skips the prereleases and returns the newest
+# stable tag, which is what darwin should pin: upstream deletes and re-cuts .dev
+# tags, and a pin to a deleted tag 404s the whole build.
 LATEST_TAG=$(gh_curl -H "Accept: application/vnd.github+json" \
     "https://api.github.com/repos/$REPO/releases/latest" \
     | python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])")
 
-# The coupled vLLM source version lives in vllm-metal's installer at that tag.
-NEW_VLLM_VERSION=$(gh_curl \
-    "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh" \
+# The coupled vLLM release lives in .github/vllm-release-tag.commit at that tag
+# (since vllm-metal 0.28); releases predating that file pinned it inline in their
+# own install.sh. The extractor reads both forms.
+NEW_VLLM_VERSION=$( { gh_curl \
+    "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/.github/vllm-release-tag.commit" \
+    || gh_curl "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh"; } \
     | "$(dirname "${BASH_SOURCE[0]}")/../scripts/lib/extract-vllm-metal-version.sh")
 
 if [ -z "$LATEST_TAG" ] || [ -z "$NEW_VLLM_VERSION" ]; then

backend/python/vllm/install.sh

@@ -119,14 +119,18 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux
     # vllm pin (requirements-cublas13-after.txt, bumped independently against
     # vllm/vllm) until vllm-metal supports a newer vLLM.
-    VLLM_METAL_VERSION="v0.3.0.dev20260818075955"
+    VLLM_METAL_VERSION="v0.28.0"
 
     # The coupled vLLM source version is whatever this vllm-metal release builds
-    # against. Derive it from
-    # the PINNED tag rather than hardcoding a second value that could drift. The
-    # tag is immutable, so this stays reproducible across rebuilds.
-    VLLM_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}/install.sh" \
-        | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh")
+    # against. Derive it from the PINNED tag rather than hardcoding a second value
+    # that could drift. The tag is immutable, so this stays reproducible across
+    # rebuilds. Since vllm-metal 0.28 the coupling is declared in
+    # .github/vllm-release-tag.commit; older releases pinned it inline in their
+    # own install.sh, so fall back to that. The extractor reads both forms.
+    _vllm_metal_raw="https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}"
+    VLLM_VERSION=$( { curl -fsSL "${_vllm_metal_raw}/.github/vllm-release-tag.commit" \
+        || curl -fsSL "${_vllm_metal_raw}/install.sh"; } \
+        | "$backend_dir/../../../scripts/lib/extract-vllm-metal-version.sh" || true)
     if [ -z "${VLLM_VERSION}" ]; then
         echo "ERROR: could not derive the vLLM version from vllm-metal ${VLLM_METAL_VERSION}" >&2
         exit 1
@@ -153,10 +157,18 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # 2) Install the prebuilt vllm-metal wheel for the PINNED release. It pulls
     #    mlx / mlx-metal as deps and registers the `metal` platform plugin that
     #    backend.py resolves to at engine-init time. Build the release-asset URL
-    #    deterministically (tag + the cp312/arm64 wheel name) rather than querying
-    #    api.github.com, whose unauthenticated rate limit (60/hr per IP) 403s on
-    #    shared CI runners. The wheel version is the tag without its leading 'v'.
-    _metal_wheel="vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-macosx_11_0_arm64.whl"
+    #    from the release's OWN asset listing rather than composing it from a
+    #    hardcoded platform tag: upstream raised its macOS deployment target
+    #    (macosx_11_0 -> macosx_15_0) and every composed URL started to 404.
+    #    expanded_assets is the plain release page, not api.github.com, whose
+    #    unauthenticated rate limit (60/hr per IP) 403s on shared CI runners.
+    #    The wheel version is the tag without its leading 'v'.
+    _metal_wheel=$(curl -fsSL "https://github.com/vllm-project/vllm-metal/releases/expanded_assets/${VLLM_METAL_VERSION}" \
+        | grep -oE "vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-[A-Za-z0-9_]+\.whl" | head -1 || true)
+    if [ -z "${_metal_wheel}" ]; then
+        echo "ERROR: no cp312 wheel asset on vllm-metal release ${VLLM_METAL_VERSION}" >&2
+        exit 1
+    fi
     _metal_wheel_url="https://github.com/vllm-project/vllm-metal/releases/download/${VLLM_METAL_VERSION}/${_metal_wheel}"
     echo "Installing vllm-metal wheel: ${_metal_wheel_url}"
     uv pip install "${_metal_wheel_url}"

scripts/build/extract-vllm-metal-version_test.sh

@@ -21,6 +21,17 @@ assert_version "0.26.0" '  local vllm_v="0.26.0"'
 assert_version "0.26.0" 'VLLM_VERSION="0.26.0"'
 assert_version "0.26.1" '  VLLM_VERSION = "0.26.1" # comment'
 
+# .github/vllm-release-tag.commit form: a lone vLLM release tag.
+assert_version "0.28.0" 'v0.28.0'
+assert_version "0.28.0" '0.28.0'
+assert_version "0.28.0" '  v0.28.0  '
+
+# A whole upstream installer must still yield the inline pin, not a version-like
+# fragment of some other line.
+assert_version "0.26.0" 'set -e
+vllm_wheel="vllm-1.2.3-cp312.whl"
+VLLM_VERSION="0.26.0"'
+
 if printf '%s\n' 'VLLM_VERSION="not-a-version"' | "$extractor"; then
     echo "malformed versions must be rejected" >&2
     exit 1
@@ -30,3 +41,13 @@ if printf '%s\n' 'VLLM_VERSION="0.26.0"garbage' | "$extractor"; then
     echo "trailing assignment content must be rejected" >&2
     exit 1
 fi
+
+if printf '%s\n' 'not-a-tag' | "$extractor"; then
+    echo "malformed release tags must be rejected" >&2
+    exit 1
+fi
+
+if printf '%s\n' 'vllm-1.2.3-cp312.whl' | "$extractor"; then
+    echo "a version embedded in a longer line must be rejected" >&2
+    exit 1
+fi

scripts/lib/extract-vllm-metal-version.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 set -euo pipefail
 
-grep -m1 -oE '^[[:space:]]*(local[[:space:]]+)?(vllm_v|VLLM_VERSION)[[:space:]]*=[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+"[[:space:]]*(#.*)?$' \
+# Print the bare X.Y.Z vLLM version a vllm-metal release builds against, reading
+# whichever form the release declares it in on stdin:
+#
+#   * .github/vllm-release-tag.commit -- a lone "vX.Y.Z" vLLM release tag. This is
+#     the source of truth since vllm-metal 0.28, which also re-versioned the
+#     project so its own version tracks the vLLM version it targets.
+#   * install.sh -- releases predating that file pinned VLLM_VERSION="X.Y.Z"
+#     (earlier still: vllm_v="X.Y.Z") inline in their installer.
+grep -m1 -oE '^[[:space:]]*((local[[:space:]]+)?(vllm_v|VLLM_VERSION)[[:space:]]*=[[:space:]]*"[0-9]+\.[0-9]+\.[0-9]+"[[:space:]]*(#.*)?|v?[0-9]+\.[0-9]+\.[0-9]+[[:space:]]*)$' \
     | grep -oE '[0-9]+\.[0-9]+\.[0-9]+'

tests/e2e-backends/backend_test.go

@@ -473,7 +473,7 @@ var _ = Describe("Backend container", Ordered, func() {
 			Expect(res.GetPromptTokens()).To(BeNumerically(">", 128),
 				"prompt is too short to span multiple prefill batches; this spec would not prove anything")
 		}
-		Expect(strings.ToUpper(res.GetMessage())).To(ContainSubstring(needle),
+		Expect(strings.ToUpper(string(res.GetMessage()))).To(ContainSubstring(needle),
 			"a long prompt lost information the model repeats correctly from a short one - "+
 				"batched prefill is corrupting state (check the backend's device architecture flags)")
 		GinkgoWriter.Printf("LongPrefill: prompt_tokens=%d tokens=%d msg=%q\n",