fix(sglang,vllm): don't cap generation at 200 tokens when max_tokens is omitted

Both Python backends start from a fixed default (sglang max_new_tokens=200,
vLLM max_tokens=200) and only replace it when the request carries Tokens.
OpenAI clients that leave max_tokens out (several agent frameworks do) were
silently truncated after 200 tokens, and the response still said
finish_reason "stop": the Go side only reports "length" against a budget
the client requested, so it cannot see a backend-side cap.

Use None instead, which allows generation up to the remaining context.
That is what sglang's own OpenAI server does when max_tokens is omitted
(the scheduler turns None into the remaining request capacity; dropping
the key would select SamplingParams' own default of 128), and vLLM treats
max_tokens=None the same way. A requested max_tokens still wins.

Tests check both cases and that the None default passes each library's own
parameter validation.

Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com>
(cherry picked from commit 5bdab54d97)

backend/python/sglang/backend.py

@@ -417,7 +417,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return backend_pb2.Result(success=False, message=str(e))
 
     def _build_sampling_params(self, request) -> dict:
-        sampling_params: dict = {"temperature": 0.7, "max_new_tokens": 200}
+        # max_new_tokens=None allows generation up to the remaining context,
+        # which is what sglang's own OpenAI server does when max_tokens is
+        # omitted (the scheduler turns None into the remaining request
+        # capacity). Leaving the key out would select SamplingParams' own
+        # default of 128 instead. A fixed number here silently truncated every
+        # request without max_tokens, and the response still reported
+        # finish_reason "stop", because the Go side only reports "length"
+        # against a budget the client requested.
+        sampling_params: dict = {"temperature": 0.7, "max_new_tokens": None}
         mapping = {
             "N": "n",
             "PresencePenalty": "presence_penalty",

backend/python/sglang/test.py

@@ -294,6 +294,52 @@ class TestSglangHelpers(unittest.TestCase):
         params = servicer._build_sampling_params(request)
         self.assertNotIn("custom_params", params)
 
+    def test_omitted_max_tokens_does_not_cap_generation(self):
+        """A request without max_tokens (Tokens=0) must not get a hidden
+        200-token cap. None makes sglang generate until the context is full,
+        as its own OpenAI server does."""
+        from types import SimpleNamespace
+
+        servicer = self._servicer()
+        request = SimpleNamespace(
+            Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0,
+            RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0,
+            StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=0,
+            MinTokens=0, SkipSpecialTokens=False, Grammar="",
+        )
+        params = servicer._build_sampling_params(request)
+        self.assertIn("max_new_tokens", params)
+        self.assertIsNone(params["max_new_tokens"])
+
+    def test_omitted_max_tokens_passes_sglang_validation(self):
+        """The None default must survive sglang's own SamplingParams checks."""
+        from types import SimpleNamespace
+        from sglang.srt.sampling.sampling_params import SamplingParams
+
+        servicer = self._servicer()
+        request = SimpleNamespace(
+            Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0,
+            RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0,
+            StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=0,
+            MinTokens=0, SkipSpecialTokens=False, Grammar="",
+        )
+        params = SamplingParams(**servicer._build_sampling_params(request))
+        params.verify(vocab_size=32000)
+        self.assertIsNone(params.max_new_tokens)
+
+    def test_requested_max_tokens_is_passed_through(self):
+        from types import SimpleNamespace
+
+        servicer = self._servicer()
+        request = SimpleNamespace(
+            Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0,
+            RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0,
+            StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=4000,
+            MinTokens=0, SkipSpecialTokens=False, Grammar="",
+        )
+        params = servicer._build_sampling_params(request)
+        self.assertEqual(params["max_new_tokens"], 4000)
+
     def test_thinking_budget_accepts_integral_spellings(self):
         """YAML options arrive as strings; "512" and "512.0" both mean 512."""
         servicer = self._servicer()

backend/python/vllm/backend.py

@@ -561,7 +561,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         # as RepeatedScalarContainer, which vLLM's argument checks and the
         # msgpack encoder to the engine core reject: every request with `stop`
         # failed. Invalid values now raise vLLM's validation error here.
-        kwargs = {"top_p": 0.9, "max_tokens": 200}
+        # max_tokens=None allows generation up to the remaining model context,
+        # which is what OpenAI clients expect when they omit max_tokens. A fixed
+        # default silently truncated those requests while still reporting
+        # finish_reason "stop".
+        kwargs = {"top_p": 0.9, "max_tokens": None}
         for request_field, param_field in request_to_sampling_params.items():
             if hasattr(request, request_field):
                 value = getattr(request, request_field)

backend/python/vllm/test.py

@@ -141,6 +141,21 @@ class TestBackendServicer(unittest.TestCase):
         self.assertEqual(sampling_params.top_p, 0.9)
         self.assertEqual(sampling_params.seed, 0)
 
+    def test_omitted_max_tokens_does_not_cap_generation(self):
+        """A request without max_tokens must not get a hidden 200-token cap;
+        None lets vLLM generate until the context is full. A requested value
+        still wins."""
+        import sys, os
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from backend import BackendServicer
+
+        servicer = BackendServicer()
+        unset = servicer._build_sampling_params(backend_pb2.PredictOptions(Prompt="hello"))
+        unset._verify_args()  # vLLM's own checks accept None
+        self.assertIsNone(unset.max_tokens)
+        requested = servicer._build_sampling_params(backend_pb2.PredictOptions(Prompt="hello", Tokens=4000))
+        self.assertEqual(requested.max_tokens, 4000)
+
 
     def test_messages_to_dicts(self):
         """