package functions_test
import (
. "github.com/mudler/LocalAI/pkg/functions"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("PEG Integration", func() {
Context("format presets", func() {
It("parses functionary format", func() {
input := `I'll help you with that.{"location": "NYC", "unit": "celsius"}`
config := FunctionsConfig{
XMLFormatPreset: "functionary",
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
})
It("parses qwen3-coder format", func() {
input := "\n\n\nNYC\n\n\ncelsius\n\n\n"
config := FunctionsConfig{
XMLFormatPreset: "qwen3-coder",
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
})
It("parses qwen3-coder format with preceding content", func() {
input := "Let me think about this...\n\n\n\nNYC\n\n\n"
config := FunctionsConfig{
XMLFormatPreset: "qwen3-coder",
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
})
It("parses minimax-m2 format", func() {
input := "Here's the result.\n\n\ntest query\n\n"
config := FunctionsConfig{
XMLFormatPreset: "minimax-m2",
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("search"))
Expect(results[0].Arguments).To(ContainSubstring(`"query"`))
})
It("handles glm-4.5 format gracefully", func() {
input := "locationNYC"
config := FunctionsConfig{
XMLFormatPreset: "glm-4.5",
}
results := ParseFunctionCallPEG(input, config)
// GLM-4.5 uses tool_call as both scope and tool start with no function name separator,
// so the PEG parser may not handle it perfectly.
if len(results) > 0 {
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
}
})
})
Context("auto-detect", func() {
It("detects format without preset", func() {
input := "\n\n\nNYC\n\n\n"
config := FunctionsConfig{}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
})
})
Context("custom XML format", func() {
It("parses with custom format config", func() {
input := "\n\n\ntest.py\n\n\nhello world\n\n\n"
config := FunctionsConfig{
XMLFormat: &XMLToolCallFormat{
ScopeStart: "",
ToolStart: "",
KeyStart: "",
ValEnd: "",
ToolEnd: "",
ScopeEnd: "",
TrimRawArgVal: true,
},
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("edit"))
Expect(results[0].Arguments).To(ContainSubstring(`"filename"`))
})
})
Context("no tool calls", func() {
It("returns empty results for plain text", func() {
input := "This is just a regular response with no tool calls."
config := FunctionsConfig{
XMLFormatPreset: "qwen3-coder",
}
results := ParseFunctionCallPEG(input, config)
Expect(results).To(BeEmpty())
})
})
Context("ParseFunctionCall integration", func() {
It("finds tool calls via PEG in ParseFunctionCall flow", func() {
input := "\n\n\nNYC\n\n\n"
config := FunctionsConfig{}
results := ParseFunctionCall(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
})
It("finds functionary tool calls via ParseFunctionCall", func() {
input := `Sure!{"expression": "2+2"}`
config := FunctionsConfig{
XMLFormatPreset: "functionary",
}
results := ParseFunctionCall(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("calculator"))
Expect(results[0].Arguments).To(ContainSubstring(`"expression"`))
})
})
Context("DisablePEGParser", func() {
It("still works when called directly but skips PEG in ParseFunctionCall", func() {
input := "\n\n\nNYC\n\n\n"
config := FunctionsConfig{
DisablePEGParser: true,
}
// ParseFunctionCallPEG should still work when called directly
pegResults := ParseFunctionCallPEG(input, config)
// May or may not find results depending on auto-detect
_ = pegResults
// ParseFunctionCall with PEG disabled should NOT find XML tool calls
disabledResults := ParseFunctionCall(input, config)
// May find via JSON extraction
_ = disabledResults
// ParseXML (iterative parser) should still find results
xmlResults, err := ParseXML(input, nil)
Expect(err).NotTo(HaveOccurred())
Expect(xmlResults).NotTo(BeEmpty())
Expect(xmlResults[0].Name).To(Equal("get_weather"))
})
})
Context("markers-based parsing", func() {
It("parses tag_with_json format from markers", func() {
input := `Hello!{"location": "NYC"}`
markers := &ToolFormatMarkers{
FormatType: "tag_with_json",
FuncNamePrefix: "",
FuncClose: "",
}
arena := BuildPEGParserFromMarkers(markers)
Expect(arena).NotTo(BeNil())
config := FunctionsConfig{
ToolFormatMarkers: markers,
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
})
It("parses tag_with_tagged format from markers", func() {
input := "\n\nNYC\n\n"
markers := &ToolFormatMarkers{
FormatType: "tag_with_tagged",
SectionStart: "",
SectionEnd: "",
FuncNamePrefix: "",
FuncClose: "",
ArgNamePrefix: "",
ArgValueSuffix: "",
}
config := FunctionsConfig{
ToolFormatMarkers: markers,
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
})
It("parses json_native format from markers", func() {
input := `Some content{"name": "get_weather", "arguments": {"location": "NYC"}}`
markers := &ToolFormatMarkers{
FormatType: "json_native",
SectionStart: "",
SectionEnd: "",
NameField: "name",
ArgsField: "arguments",
}
config := FunctionsConfig{
ToolFormatMarkers: markers,
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
})
It("returns nil arena for unknown format type", func() {
markers := &ToolFormatMarkers{
FormatType: "unknown",
}
arena := BuildPEGParserFromMarkers(markers)
Expect(arena).To(BeNil())
})
It("parses json_native format with ID field", func() {
input := `Some content{"name": "get_weather", "arguments": {"location": "NYC"}, "id": "call_123"}`
markers := &ToolFormatMarkers{
FormatType: "json_native",
SectionStart: "",
SectionEnd: "",
NameField: "name",
ArgsField: "arguments",
IDField: "id",
}
config := FunctionsConfig{
ToolFormatMarkers: markers,
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
Expect(results[0].ID).To(Equal("call_123"))
})
It("parses call ID between function name and arguments", func() {
input := `[call_abc]{"location": "NYC"}`
markers := &ToolFormatMarkers{
FormatType: "tag_with_json",
SectionStart: "",
SectionEnd: "",
FuncNamePrefix: "",
FuncClose: "",
CallIDPosition: "between_func_and_args",
CallIDPrefix: "[",
CallIDSuffix: "]",
}
config := FunctionsConfig{
ToolFormatMarkers: markers,
}
results := ParseFunctionCallPEG(input, config)
Expect(results).NotTo(BeEmpty())
Expect(results[0].Name).To(Equal("get_weather"))
Expect(results[0].ID).To(Equal("call_abc"))
Expect(results[0].Arguments).To(ContainSubstring(`"location"`))
})
})
})