mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-30 09:57:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(vllm): apply Options[] engine flags before engine init (#11147)

Browse Source 
fix(vllm): apply Options[] engine flags before engine init (#11130)

CLI-style flags in a model's `options:` array (`--quantization:gptq_marlin`,
`--enable-prefix-caching`, `--kv-cache-dtype:fp8_e5m2`) were discarded: the
backend only ever read `tool_parser`/`reasoning_parser` out of Options[], and
did so *after* `AsyncLLMEngine.from_engine_args()`, where nothing it set could
still reach the engine.

Map `--` prefixed options onto the AsyncEngineArgs dataclass before the engine
is constructed. Names are normalized the way vLLM's CLI spells them
(`--enable-prefix-caching` -> `enable_prefix_caching`), values are coerced to
the target field's type (bare flag -> True for booleans), and unknown or
uncoercible flags warn and are skipped instead of failing the load, since
Options[] is a bag shared with backend-level settings. Field types come from
the annotation's base so `Literal["auto", "float16"]` (vLLM's dtype) is not
mistaken for a float.

Precedence is typed proto fields -> `options:` -> `engine_args:`. The
production engine_args defaults seeded in hooks_vllm.go therefore skip any key
the user already set as an option, otherwise the later engine_args pass would
silently override it. Parser lookups now accept both spellings, so
`--reasoning-parser:qwen3` selects LocalAI's parser as well.

The helper's tests are stdlib-only and run in the lint workflow's
dependency-light job via `make test-python-helpers`.


Assisted-by: Claude:claude-opus-5 golangci-lint

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: localai-org-maint-bot <bot-opensource@localaisrl.com>
This commit is contained in:
mudler's LocalAI [bot]
2026-07-30 12:13:00 +02:00
committed by GitHub
parent 9c85cacfe3
commit 47097041ff
9 changed files with 481 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
backend/python/common/vllm_utils.py
View File

@@ -4,11 +4,208 @@ Generic helpers (``parse_options``, ``messages_to_dicts``) live in
``python_utils`` and are re-exported here for backwards compatibility with
existing imports in both backends.
"""
import dataclasses
import difflib
import json
import sys
from python_utils import messages_to_dicts, parse_options
__all__ = ["parse_options", "messages_to_dicts", "setup_parsers"]
__all__ = [
"parse_options",
"messages_to_dicts",
"setup_parsers",
"apply_options_to_engine_args",
"normalize_option_key",
]
# Options[] entries LocalAI itself acts on, in their normalized spelling.
_BACKEND_LEVEL_OPTIONS = {"tool_parser", "reasoning_parser"}
_TRUTHY = {"true", "1", "yes", "on", "t", "y"}
_FALSY = {"false", "0", "no", "off", "f", "n"}
def normalize_option_key(key):
"""Normalize an Options[] key to its engine/field spelling.
Users copy flags straight out of vLLM's CLI docs (``--reasoning-parser``),
so accept that spelling everywhere LocalAI looks options up by name.
"""
return key.strip().lstrip("-").replace("-", "_")
_BASE_HINTS = {
"bool": "bool",
"int": "int",
"float": "float",
"str": "str",
"dict": "dict",
"mapping": "dict",
}
def _hint_from_annotation(annotation):
"""Target type name for a field annotation, or None when it is not a scalar.
``dataclasses.fields()`` hands back either a real type, a ``typing``
construct or - under PEP 563, which vLLM's arg_utils uses - the annotation
as a plain string, so work on the textual form. Match the *base* of the
annotation rather than searching it: ``Literal["auto", "float16"]`` (vLLM's
dtype) contains "float" but is not a float.
"""
if annotation is None:
return None
if isinstance(annotation, type):
text = annotation.__name__
elif isinstance(annotation, str):
text = annotation
else:
text = str(annotation)
text = text.replace("typing.", "").strip()
while text.lower().startswith("optional[") and text.endswith("]"):
text = text[len("optional["):-1].strip()
if "|" in text:
parts = [p.strip() for p in text.split("|") if p.strip().lower() not in ("none", "nonetype")]
if len(parts) != 1:
return None
text = parts[0]
base = text.split("[", 1)[0].strip().lower()
return _BASE_HINTS.get(base)
def _hint_from_value(current):
"""Target type name inferred from a field's current value."""
if isinstance(current, bool):
return "bool"
if isinstance(current, dict):
return "dict"
if isinstance(current, float):
return "float"
if isinstance(current, int):
return "int"
if isinstance(current, str):
return "str"
return None
def _type_hint(annotation, current):
"""Best-effort target type name for a dataclass field."""
hint = _hint_from_annotation(annotation)
if hint is None:
hint = _hint_from_value(current)
return hint
def _coerce_option(raw, hint):
"""Coerce a CLI-supplied string to the field's type. Raises ValueError."""
if hint == "bool":
low = raw.lower()
if low in _TRUTHY:
return True
if low in _FALSY:
return False
raise ValueError(f"{raw!r} is not a boolean")
if hint == "int":
return int(raw)
if hint == "float":
return float(raw)
if hint == "dict":
return json.loads(raw)
if hint == "str":
return raw
# Untyped (or union-typed) field: infer from the literal itself.
low = raw.lower()
if low in _TRUTHY:
return True
if low in _FALSY:
return False
for cast in (int, float):
try:
return cast(raw)
except ValueError:
pass
if raw.startswith("{") or raw.startswith("["):
return json.loads(raw)
return raw
def apply_options_to_engine_args(engine_args, options):
"""Apply CLI-style ``--flag[:value]`` entries from Options[] to engine args.
``options:`` is a loose bag shared with backend-level settings
(``tool_parser:``, ``reasoning_parser:``, ``vad_only``, …), so only
``--`` prefixed entries are treated as engine flags. Names are normalized
the way vLLM's own CLI does (``--enable-prefix-caching`` →
``enable_prefix_caching``) and values are coerced to the target field's
type. Both ``--flag:value`` (LocalAI convention) and ``--flag=value``
(vLLM CLI convention) are accepted; a bare ``--flag`` sets a boolean field.
Unknown or uncoercible flags warn on stderr and are skipped instead of
failing the load - unlike ``engine_args:``, which is engine-only and
therefore strict, Options[] carries entries this function knows nothing
about.
Returns a new dataclass instance via ``dataclasses.replace`` so the
engine's ``__post_init__`` re-runs, or the original when nothing applied.
"""
if not options:
return engine_args
fields = {f.name: f for f in dataclasses.fields(type(engine_args))}
updates = {}
for opt in options:
opt = opt.strip()
if not opt.startswith("--"):
continue
body = opt[2:]
seps = [i for i in (body.find(":"), body.find("=")) if i != -1]
if seps:
cut = min(seps)
name, raw = body[:cut], body[cut + 1:].strip()
else:
name, raw = body, None
name = normalize_option_key(name)
if name not in fields:
# LocalAI reads these itself; whether the engine dataclass carries
# them too varies by vLLM version, so never flag them as unknown.
if name in _BACKEND_LEVEL_OPTIONS:
continue
suggestion = difflib.get_close_matches(name, fields, n=1)
hint = f" did you mean {suggestion[0]!r}?" if suggestion else ""
print(
f"[vllm_utils] unknown engine option {opt!r} (field {name!r}), skipping.{hint}",
file=sys.stderr,
)
continue
target = _type_hint(fields[name].type, getattr(engine_args, name, None))
if raw is None:
if target != "bool":
print(
f"[vllm_utils] engine option {opt!r} needs a value "
f"(field {name!r} is not a flag), skipping",
file=sys.stderr,
)
continue
updates[name] = True
continue
try:
updates[name] = _coerce_option(raw, target)
except (ValueError, TypeError) as err:
print(
f"[vllm_utils] cannot apply engine option {opt!r} to field "
f"{name!r}: {err}, skipping",
file=sys.stderr,
)
if not updates:
return engine_args
print(f"[vllm_utils] engine options from Options[]: {updates}", file=sys.stderr)
return dataclasses.replace(engine_args, **updates)
def setup_parsers(opts):

160
backend/python/common/vllm_utils_test.py Normal file
View File

@@ -0,0 +1,160 @@
"""Unit tests for the shared vLLM backend helpers (vllm_utils.py).
Run standalone (Python standard library only, no backend venv needed):
cd backend/python/common && python3 -m unittest vllm_utils_test
``vllm_utils`` imports vLLM lazily (inside functions), so the module is
importable without the vLLM wheel. ``AsyncEngineArgs`` is stood in for by a
local dataclass that mirrors the field shapes that matter: plain scalars,
``Optional[...]`` scalars (vLLM's tri-state flags) and dict-valued fields.
"""
import contextlib
import dataclasses
import io
import unittest
from typing import Dict, Literal, Optional, Union
from vllm_utils import apply_options_to_engine_args, normalize_option_key
@dataclasses.dataclass
class FakeEngineArgs:
model: str = ""
quantization: Optional[str] = None
kv_cache_dtype: str = "auto"
enable_prefix_caching: Optional[bool] = None
enforce_eager: bool = False
max_model_len: Optional[int] = None
tensor_parallel_size: int = 1
gpu_memory_utilization: float = 0.9
limit_mm_per_prompt: Optional[Dict[str, int]] = None
# vLLM types dtype as a Literal of strings, several of which contain the
# word "float" - a naive match reads that as a float field.
dtype: Literal["auto", "float16", "bfloat16"] = "auto"
seed: Union[int, str, None] = None
# vLLM's arg_utils is annotated under PEP 563, where dataclasses.fields() hands
# back annotations as plain strings rather than types.
StringAnnotatedEngineArgs = dataclasses.make_dataclass(
"StringAnnotatedEngineArgs",
[
("max_model_len", "int | None", dataclasses.field(default=None)),
("enable_prefix_caching", "Optional[bool]", dataclasses.field(default=None)),
("kv_cache_dtype", "str", dataclasses.field(default="auto")),
],
)
def _apply(options, **overrides):
"""Apply options to a fresh FakeEngineArgs, returning (result, stderr)."""
err = io.StringIO()
with contextlib.redirect_stderr(err):
out = apply_options_to_engine_args(FakeEngineArgs(**overrides), options)
return out, err.getvalue()
class TestApplyOptionsToEngineArgs(unittest.TestCase):
def test_string_option_is_applied(self):
out, _ = _apply(["--quantization:gptq_marlin"])
self.assertEqual(out.quantization, "gptq_marlin")
def test_valueless_flag_enables_boolean_field(self):
out, _ = _apply(["--enable-prefix-caching"])
self.assertIs(out.enable_prefix_caching, True)
def test_boolean_field_accepts_explicit_false(self):
out, _ = _apply(["--enable-prefix-caching:false"], enable_prefix_caching=True)
self.assertIs(out.enable_prefix_caching, False)
def test_integer_field_is_coerced(self):
out, _ = _apply(["--max-model-len:4096", "--tensor-parallel-size:2"])
self.assertEqual(out.max_model_len, 4096)
self.assertEqual(out.tensor_parallel_size, 2)
def test_float_field_is_coerced(self):
out, _ = _apply(["--gpu-memory-utilization:0.85"])
self.assertAlmostEqual(out.gpu_memory_utilization, 0.85)
def test_equals_separator_is_accepted(self):
out, _ = _apply(["--kv-cache-dtype=fp8_e5m2"])
self.assertEqual(out.kv_cache_dtype, "fp8_e5m2")
def test_dict_field_is_parsed_as_json(self):
out, _ = _apply(['--limit-mm-per-prompt:{"image": 4}'])
self.assertEqual(out.limit_mm_per_prompt, {"image": 4})
def test_non_flag_options_are_left_alone(self):
out, err = _apply(["tool_parser:hermes", "reasoning_parser:qwen3", "vad_only"])
self.assertEqual(out, FakeEngineArgs())
self.assertEqual(err, "")
def test_unknown_flag_warns_and_is_skipped(self):
out, err = _apply(["--not-a-real-flag:1", "--quantization:awq"])
self.assertEqual(out.quantization, "awq")
self.assertIn("not_a_real_flag", err)
self.assertIn("unknown", err.lower())
def test_parser_flags_are_not_reported_as_unknown(self):
# LocalAI consumes these itself; whether the engine dataclass also has
# the field depends on the vLLM version, and warning about them would
# send users chasing a non-problem.
out, err = _apply(["--tool-parser:hermes", "--reasoning-parser:qwen3"])
self.assertEqual(out, FakeEngineArgs())
self.assertEqual(err, "")
def test_unknown_flag_hints_at_the_closest_field(self):
_, err = _apply(["--max-model-length:4096"])
self.assertIn("max_model_len", err)
def test_uncoercible_value_warns_and_is_skipped(self):
out, err = _apply(["--max-model-len:lots"])
self.assertIsNone(out.max_model_len)
self.assertIn("max_model_len", err)
def test_valueless_flag_on_non_boolean_field_warns_and_is_skipped(self):
out, err = _apply(["--quantization"])
self.assertIsNone(out.quantization)
self.assertIn("quantization", err)
def test_empty_options_returns_the_same_engine_args(self):
original = FakeEngineArgs(model="m")
self.assertIs(apply_options_to_engine_args(original, []), original)
class TestFieldTypeInference(unittest.TestCase):
def test_literal_typed_field_keeps_its_string_value(self):
out, err = _apply(["--dtype:bfloat16"])
self.assertEqual(out.dtype, "bfloat16")
self.assertEqual(err.count("skipping"), 0)
def test_ambiguous_union_falls_back_to_literal_inference(self):
out, _ = _apply(["--seed:42"])
self.assertEqual(out.seed, 42)
def test_string_annotations_are_understood(self):
err = io.StringIO()
with contextlib.redirect_stderr(err):
out = apply_options_to_engine_args(
StringAnnotatedEngineArgs(),
["--max-model-len:4096", "--enable-prefix-caching", "--kv-cache-dtype:fp8_e5m2"],
)
self.assertEqual(out.max_model_len, 4096)
self.assertIs(out.enable_prefix_caching, True)
self.assertEqual(out.kv_cache_dtype, "fp8_e5m2")
class TestNormalizeOptionKey(unittest.TestCase):
def test_cli_flag_becomes_a_field_name(self):
self.assertEqual(normalize_option_key("--reasoning-parser"), "reasoning_parser")
def test_plain_key_is_untouched(self):
self.assertEqual(normalize_option_key("tool_parser"), "tool_parser")
def test_surrounding_whitespace_is_stripped(self):
self.assertEqual(normalize_option_key(" --tool-parser "), "tool_parser")
if __name__ == "__main__":
unittest.main()

16
backend/python/vllm/backend.py
View File

@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
from grpc_auth import get_auth_interceptors
from model_utils import resolve_model_reference
from vllm_utils import apply_options_to_engine_args, normalize_option_key
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -101,13 +102,18 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
return decoded_text
def _parse_options(self, options_list):
"""Parse Options[] key:value string list into a dict."""
"""Parse Options[] key:value string list into a dict.
Keys are normalized to their field spelling so the CLI form users copy
from vLLM's docs (``--reasoning-parser:qwen3``) selects the same parser
as LocalAI's own (``reasoning_parser:qwen3``).
"""
opts = {}
for opt in options_list:
if ":" not in opt:
continue
key, value = opt.split(":", 1)
opts[key.strip()] = value.strip()
opts[normalize_option_key(key)] = value.strip()
return opts
def _apply_engine_args(self, engine_args, engine_args_json):
@@ -229,6 +235,12 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
"audio": max(request.LimitAudioPerPrompt, 1)
}
# CLI-style flags in options: (--quantization:gptq_marlin,
# --enable-prefix-caching, ...) land on the engine args too - they must
# be applied *before* the engine is created or they do nothing.
# engine_args: is applied after this, so it stays the last word.
engine_args = apply_options_to_engine_args(engine_args, request.Options)
# engine_args from YAML overrides typed fields above so operators can
# tune anything the AsyncEngineArgs dataclass exposes without waiting
# on protobuf changes.