package functions_test import ( . "github.com/mudler/LocalAI/pkg/functions" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("PEG Integration", func() { Context("format presets", func() { It("parses functionary format", func() { input := `I'll help you with that.{"location": "NYC", "unit": "celsius"}` config := FunctionsConfig{ XMLFormatPreset: "functionary", } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) }) It("parses qwen3-coder format", func() { input := "\n\n\nNYC\n\n\ncelsius\n\n\n" config := FunctionsConfig{ XMLFormatPreset: "qwen3-coder", } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) }) It("parses qwen3-coder format with preceding content", func() { input := "Let me think about this...\n\n\n\nNYC\n\n\n" config := FunctionsConfig{ XMLFormatPreset: "qwen3-coder", } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) }) It("parses minimax-m2 format", func() { input := "Here's the result.\n\n\ntest query\n\n" config := FunctionsConfig{ XMLFormatPreset: "minimax-m2", } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("search")) Expect(results[0].Arguments).To(ContainSubstring(`"query"`)) }) It("handles glm-4.5 format gracefully", func() { input := "locationNYC" config := FunctionsConfig{ XMLFormatPreset: "glm-4.5", } results := ParseFunctionCallPEG(input, config) // GLM-4.5 uses tool_call as both scope and tool start with no function name separator, // so the PEG parser may not handle it perfectly. if len(results) > 0 { Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) } }) }) Context("auto-detect", func() { It("detects format without preset", func() { input := "\n\n\nNYC\n\n\n" config := FunctionsConfig{} results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) }) }) Context("custom XML format", func() { It("parses with custom format config", func() { input := "\n\n\ntest.py\n\n\nhello world\n\n\n" config := FunctionsConfig{ XMLFormat: &XMLToolCallFormat{ ScopeStart: "", ToolStart: "", KeyStart: "", ValEnd: "", ToolEnd: "", ScopeEnd: "", TrimRawArgVal: true, }, } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("edit")) Expect(results[0].Arguments).To(ContainSubstring(`"filename"`)) }) }) Context("no tool calls", func() { It("returns empty results for plain text", func() { input := "This is just a regular response with no tool calls." config := FunctionsConfig{ XMLFormatPreset: "qwen3-coder", } results := ParseFunctionCallPEG(input, config) Expect(results).To(BeEmpty()) }) }) Context("ParseFunctionCall integration", func() { It("finds tool calls via PEG in ParseFunctionCall flow", func() { input := "\n\n\nNYC\n\n\n" config := FunctionsConfig{} results := ParseFunctionCall(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) }) It("finds functionary tool calls via ParseFunctionCall", func() { input := `Sure!{"expression": "2+2"}` config := FunctionsConfig{ XMLFormatPreset: "functionary", } results := ParseFunctionCall(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("calculator")) Expect(results[0].Arguments).To(ContainSubstring(`"expression"`)) }) }) Context("DisablePEGParser", func() { It("still works when called directly but skips PEG in ParseFunctionCall", func() { input := "\n\n\nNYC\n\n\n" config := FunctionsConfig{ DisablePEGParser: true, } // ParseFunctionCallPEG should still work when called directly pegResults := ParseFunctionCallPEG(input, config) // May or may not find results depending on auto-detect _ = pegResults // ParseFunctionCall with PEG disabled should NOT find XML tool calls disabledResults := ParseFunctionCall(input, config) // May find via JSON extraction _ = disabledResults // ParseXML (iterative parser) should still find results xmlResults, err := ParseXML(input, nil) Expect(err).NotTo(HaveOccurred()) Expect(xmlResults).NotTo(BeEmpty()) Expect(xmlResults[0].Name).To(Equal("get_weather")) }) }) Context("markers-based parsing", func() { It("parses tag_with_json format from markers", func() { input := `Hello!{"location": "NYC"}` markers := &ToolFormatMarkers{ FormatType: "tag_with_json", FuncNamePrefix: "", FuncClose: "", } arena := BuildPEGParserFromMarkers(markers) Expect(arena).NotTo(BeNil()) config := FunctionsConfig{ ToolFormatMarkers: markers, } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) }) It("parses tag_with_tagged format from markers", func() { input := "\n\nNYC\n\n" markers := &ToolFormatMarkers{ FormatType: "tag_with_tagged", SectionStart: "", SectionEnd: "", FuncNamePrefix: "", FuncClose: "", ArgNamePrefix: "", ArgValueSuffix: "", } config := FunctionsConfig{ ToolFormatMarkers: markers, } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) }) It("parses json_native format from markers", func() { input := `Some content{"name": "get_weather", "arguments": {"location": "NYC"}}` markers := &ToolFormatMarkers{ FormatType: "json_native", SectionStart: "", SectionEnd: "", NameField: "name", ArgsField: "arguments", } config := FunctionsConfig{ ToolFormatMarkers: markers, } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) }) It("returns nil arena for unknown format type", func() { markers := &ToolFormatMarkers{ FormatType: "unknown", } arena := BuildPEGParserFromMarkers(markers) Expect(arena).To(BeNil()) }) It("parses json_native format with ID field", func() { input := `Some content{"name": "get_weather", "arguments": {"location": "NYC"}, "id": "call_123"}` markers := &ToolFormatMarkers{ FormatType: "json_native", SectionStart: "", SectionEnd: "", NameField: "name", ArgsField: "arguments", IDField: "id", } config := FunctionsConfig{ ToolFormatMarkers: markers, } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) Expect(results[0].ID).To(Equal("call_123")) }) It("parses call ID between function name and arguments", func() { input := `[call_abc]{"location": "NYC"}` markers := &ToolFormatMarkers{ FormatType: "tag_with_json", SectionStart: "", SectionEnd: "", FuncNamePrefix: "", FuncClose: "", CallIDPosition: "between_func_and_args", CallIDPrefix: "[", CallIDSuffix: "]", } config := FunctionsConfig{ ToolFormatMarkers: markers, } results := ParseFunctionCallPEG(input, config) Expect(results).NotTo(BeEmpty()) Expect(results[0].Name).To(Equal("get_weather")) Expect(results[0].ID).To(Equal("call_abc")) Expect(results[0].Arguments).To(ContainSubstring(`"location"`)) }) }) })