diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index f863631f6..8309bee10 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -696,6 +696,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator template = predInput } thinkingStartToken := reason.DetectThinkingStartToken(template, &config.ReasoningConfig) + if config.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig) + } xlog.Debug("Thinking start token", "thinkingStartToken", thinkingStartToken, "template", template) diff --git a/core/http/endpoints/openai/chat_stream_workers.go b/core/http/endpoints/openai/chat_stream_workers.go index 839f40676..f013b39a7 100644 --- a/core/http/endpoints/openai/chat_stream_workers.go +++ b/core/http/endpoints/openai/chat_stream_workers.go @@ -150,6 +150,9 @@ func processStream( template = s } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig) // preferAutoparser is sticky: once the C++ autoparser has ever classified @@ -248,6 +251,9 @@ func processStreamWithTools( template = prompt } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig) result := "" diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index bee3b52ed..28730a447 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2480,6 +2480,9 @@ func triggerResponseAtTurn(ctx context.Context, session *Session, conv *Conversa template = config.TemplateConfig.Chat } thinkingStartToken := reasoning.DetectThinkingStartToken(template, &config.ReasoningConfig) + if config.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reasoning.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig) + } // When the C++ autoparser emitted ChatDeltas with actionable data, // prefer them — the backend clears Reply.Message in that path and diff --git a/core/http/endpoints/openai/realtime_stream.go b/core/http/endpoints/openai/realtime_stream.go index 7c37f7aff..debe2de2b 100644 --- a/core/http/endpoints/openai/realtime_stream.go +++ b/core/http/endpoints/openai/realtime_stream.go @@ -145,6 +145,9 @@ func streamLLMResponse(ctx context.Context, session *Session, conv *Conversation template = llmCfg.TemplateConfig.Chat } thinkingStartToken := reasoning.DetectThinkingStartToken(template, &llmCfg.ReasoningConfig) + if llmCfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reasoning.DetectThinkingStartTokenInTemplate(template, &llmCfg.ReasoningConfig) + } // The autoparser (tokenizer-template path) already delivers reasoning-free // content. Prefilling the thinking start token here would re-tag that clean diff --git a/core/http/endpoints/openresponses/responses.go b/core/http/endpoints/openresponses/responses.go index d86780fbd..d110ecf8f 100644 --- a/core/http/endpoints/openresponses/responses.go +++ b/core/http/endpoints/openresponses/responses.go @@ -1359,6 +1359,9 @@ func handleOpenResponsesNonStream(c echo.Context, responseID string, createdAt i template = predInput } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } // Extract reasoning from result before cleaning reasoningContent, cleanedResult := reason.ExtractReasoningComplete(result, thinkingStartToken, cfg.ReasoningConfig) @@ -1640,6 +1643,9 @@ func handleOpenResponsesStream(c echo.Context, responseID string, createdAt int6 template = predInput } thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) + if cfg.TemplateConfig.UseTokenizerTemplate { + thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } // Track state for streaming var currentMessageID string diff --git a/pkg/reasoning/reasoning.go b/pkg/reasoning/reasoning.go index 108276c3d..e6a03288e 100644 --- a/pkg/reasoning/reasoning.go +++ b/pkg/reasoning/reasoning.go @@ -21,6 +21,17 @@ import ( // - [THINK] (Magistral models) // Custom tokens from config are checked first, then default tokens. func DetectThinkingStartToken(prompt string, config *Config) string { + return detectThinkingStartToken(prompt, config, true) +} + +// DetectThinkingStartTokenInTemplate detects a possible prefill in an +// unrendered tokenizer template. Marker ordering cannot reveal which Jinja +// branch will render, so matching closing markers are intentionally ignored. +func DetectThinkingStartTokenInTemplate(template string, config *Config) string { + return detectThinkingStartToken(template, config, false) +} + +func detectThinkingStartToken(prompt string, config *Config, honorClosingToken bool) string { // Common thinking start tokens (in order of specificity - longer first) // Based on llama.cpp's chat-parser.cpp implementations defaultTokens := []string{ @@ -44,7 +55,8 @@ func DetectThinkingStartToken(prompt string, config *Config) string { // Check if prompt ends with any of these tokens (allowing for trailing whitespace/newlines) trimmedPrompt := strings.TrimRight(prompt, " \t\n\r") for _, token := range thinkingStartTokens { - if strings.Contains(trimmedPrompt, token) { + if strings.Contains(trimmedPrompt, token) && + (!honorClosingToken || !thinkingTokenClosedAfterLastStart(trimmedPrompt, token, config)) { return token } } @@ -67,6 +79,15 @@ func DetectThinkingStartToken(prompt string, config *Config) string { return "" } +func thinkingTokenClosedAfterLastStart(prompt, startToken string, config *Config) bool { + endToken := ClosingTokenForStart(startToken, config) + if endToken == "" { + return false + } + + return strings.LastIndex(prompt, endToken) > strings.LastIndex(prompt, startToken) +} + // ExtractReasoningWithConfig extracts reasoning from content with the given config. // If reasoning is disabled, it returns the original content. // If thinking start token prefill is enabled, it prepends the thinking start token to the content. diff --git a/pkg/reasoning/reasoning_test.go b/pkg/reasoning/reasoning_test.go index 5e6151b01..c58d34a59 100644 --- a/pkg/reasoning/reasoning_test.go +++ b/pkg/reasoning/reasoning_test.go @@ -410,6 +410,29 @@ var _ = Describe("DetectThinkingStartToken", func() { token := DetectThinkingStartToken(prompt, nil) Expect(token).To(Equal("")) }) + + It("should ignore a Gemma thinking token that is already closed in the prompt", func() { + prompt := "<|turn>model\n<|channel>thought\n\n" + token := DetectThinkingStartToken(prompt, nil) + Expect(token).To(BeEmpty()) + + extractor := NewReasoningExtractor(token, Config{}) + reasoningDelta, contentDelta := extractor.ProcessToken("READY.") + Expect(reasoningDelta).To(BeEmpty()) + Expect(contentDelta).To(Equal("READY.")) + }) + + It("should preserve prefill detection for unrendered conditional templates", func() { + template := "{% if enable_thinking %}{% else %}{% endif %}" + token := DetectThinkingStartTokenInTemplate(template, nil) + Expect(token).To(Equal("")) + }) + + It("should ignore user Jinja text before a preclosed Gemma prompt suffix", func() { + prompt := "Explain {{ variable }}\n<|turn>model\n<|channel>thought\n\n" + token := DetectThinkingStartToken(prompt, nil) + Expect(token).To(BeEmpty()) + }) }) Context("when prompt does not contain thinking tokens", func() {