diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index 7936ddcf2..b3a41a2ce 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -417,7 +417,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): return backend_pb2.Result(success=False, message=str(e)) def _build_sampling_params(self, request) -> dict: - sampling_params: dict = {"temperature": 0.7, "max_new_tokens": 200} + # max_new_tokens=None allows generation up to the remaining context, + # which is what sglang's own OpenAI server does when max_tokens is + # omitted (the scheduler turns None into the remaining request + # capacity). Leaving the key out would select SamplingParams' own + # default of 128 instead. A fixed number here silently truncated every + # request without max_tokens, and the response still reported + # finish_reason "stop", because the Go side only reports "length" + # against a budget the client requested. + sampling_params: dict = {"temperature": 0.7, "max_new_tokens": None} mapping = { "N": "n", "PresencePenalty": "presence_penalty", diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 1b1a10d23..f99774770 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -294,6 +294,52 @@ class TestSglangHelpers(unittest.TestCase): params = servicer._build_sampling_params(request) self.assertNotIn("custom_params", params) + def test_omitted_max_tokens_does_not_cap_generation(self): + """A request without max_tokens (Tokens=0) must not get a hidden + 200-token cap. None makes sglang generate until the context is full, + as its own OpenAI server does.""" + from types import SimpleNamespace + + servicer = self._servicer() + request = SimpleNamespace( + Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0, + RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0, + StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=0, + MinTokens=0, SkipSpecialTokens=False, Grammar="", + ) + params = servicer._build_sampling_params(request) + self.assertIn("max_new_tokens", params) + self.assertIsNone(params["max_new_tokens"]) + + def test_omitted_max_tokens_passes_sglang_validation(self): + """The None default must survive sglang's own SamplingParams checks.""" + from types import SimpleNamespace + from sglang.srt.sampling.sampling_params import SamplingParams + + servicer = self._servicer() + request = SimpleNamespace( + Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0, + RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0, + StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=0, + MinTokens=0, SkipSpecialTokens=False, Grammar="", + ) + params = SamplingParams(**servicer._build_sampling_params(request)) + params.verify(vocab_size=32000) + self.assertIsNone(params.max_new_tokens) + + def test_requested_max_tokens_is_passed_through(self): + from types import SimpleNamespace + + servicer = self._servicer() + request = SimpleNamespace( + Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0, + RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0, + StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=4000, + MinTokens=0, SkipSpecialTokens=False, Grammar="", + ) + params = servicer._build_sampling_params(request) + self.assertEqual(params["max_new_tokens"], 4000) + def test_thinking_budget_accepts_integral_spellings(self): """YAML options arrive as strings; "512" and "512.0" both mean 512.""" servicer = self._servicer() diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 15301a0b7..9c89a1ec5 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -561,7 +561,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # as RepeatedScalarContainer, which vLLM's argument checks and the # msgpack encoder to the engine core reject: every request with `stop` # failed. Invalid values now raise vLLM's validation error here. - kwargs = {"top_p": 0.9, "max_tokens": 200} + # max_tokens=None allows generation up to the remaining model context, + # which is what OpenAI clients expect when they omit max_tokens. A fixed + # default silently truncated those requests while still reporting + # finish_reason "stop". + kwargs = {"top_p": 0.9, "max_tokens": None} for request_field, param_field in request_to_sampling_params.items(): if hasattr(request, request_field): value = getattr(request, request_field) diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index 6317955c0..6fd560df8 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -141,6 +141,21 @@ class TestBackendServicer(unittest.TestCase): self.assertEqual(sampling_params.top_p, 0.9) self.assertEqual(sampling_params.seed, 0) + def test_omitted_max_tokens_does_not_cap_generation(self): + """A request without max_tokens must not get a hidden 200-token cap; + None lets vLLM generate until the context is full. A requested value + still wins.""" + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + + servicer = BackendServicer() + unset = servicer._build_sampling_params(backend_pb2.PredictOptions(Prompt="hello")) + unset._verify_args() # vLLM's own checks accept None + self.assertIsNone(unset.max_tokens) + requested = servicer._build_sampling_params(backend_pb2.PredictOptions(Prompt="hello", Tokens=4000)) + self.assertEqual(requested.max_tokens, 4000) + def test_messages_to_dicts(self): """