package peg_test import ( "encoding/json" "github.com/mudler/LocalAI/pkg/functions/peg" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("PEG Utils", func() { Context("peg.NormalizeQuotesToJSON", func() { It("converts basic single quotes to double quotes", func() { input := "{'key': 'value'}" expected := `{"key": "value"}` Expect(peg.NormalizeQuotesToJSON(input)).To(Equal(expected)) }) It("handles escaped single quotes", func() { input := `{'code': 'print(\'hello\')'}` expected := `{"code": "print('hello')"}` Expect(peg.NormalizeQuotesToJSON(input)).To(Equal(expected)) }) It("handles double quotes inside single-quoted strings", func() { input := `{'msg': 'He said "hi"'}` expected := `{"msg": "He said \"hi\""}` Expect(peg.NormalizeQuotesToJSON(input)).To(Equal(expected)) }) It("handles nested backslash escapes", func() { input := `{'path': 'C:\\Users\\test'}` expected := `{"path": "C:\\Users\\test"}` Expect(peg.NormalizeQuotesToJSON(input)).To(Equal(expected)) }) It("handles newline escapes", func() { input := `{'text': 'line1\nline2'}` expected := `{"text": "line1\nline2"}` Expect(peg.NormalizeQuotesToJSON(input)).To(Equal(expected)) }) It("handles mixed quotes", func() { input := `{"already_double": 'single_value'}` expected := `{"already_double": "single_value"}` Expect(peg.NormalizeQuotesToJSON(input)).To(Equal(expected)) }) It("handles embedded quotes complex case", func() { input := `{'filename': 'foo.cpp', 'oldString': 'def foo(arg = "14"):\n return arg + "bar"\n', 'newString': 'def foo(arg = "15"):\n pass\n'}` result := peg.NormalizeQuotesToJSON(input) var parsed map[string]string err := json.Unmarshal([]byte(result), &parsed) Expect(err).NotTo(HaveOccurred(), "result is not valid JSON: %s", result) Expect(parsed["filename"]).To(Equal("foo.cpp")) Expect(parsed["oldString"]).NotTo(BeEmpty()) }) }) Context("peg.EscapeJSONStringInner", func() { It("leaves basic strings unchanged", func() { Expect(peg.EscapeJSONStringInner("hello")).To(Equal("hello")) }) It("escapes double quotes", func() { Expect(peg.EscapeJSONStringInner(`hello "world"`)).To(Equal(`hello \"world\"`)) }) It("escapes backslash-n sequences", func() { Expect(peg.EscapeJSONStringInner(`line1\nline2`)).To(Equal(`line1\\nline2`)) }) }) Context("StandardJSONTools OpenAI format", func() { It("parses OpenAI-style tool calls with call ID", func() { tools := []peg.ToolDef{ { Name: "get_current_weather", Properties: map[string]peg.PropDef{ "location": {Type: "string"}, }, }, } parser := peg.BuildChatPegParser(func(p *peg.ChatBuilder) peg.ParserID { toolCall := p.StandardJSONTools(peg.StandardJSONToolsOpts{ SectionStart: "", SectionEnd: "", Tools: tools, CallIDKey: "id", ParametersOrder: []string{"id", "name", "arguments"}, }) return p.Seq( p.Content(p.Until("")), p.Optional(p.Seq(p.Space(), toolCall)), p.End(), ) }) input := `Let me check the weather.{"id": "call_abc123", "name": "get_current_weather", "arguments": {"location": "NYC"}}` ctx := peg.NewParseContext(input, false) result := parser.Parse(ctx) Expect(result.Type).To(Equal(peg.Success)) mapper := &peg.ChatPegMapper{} mapper.FromAST(&ctx.Ast, &result) msg := mapper.Result Expect(msg.ToolCalls).To(HaveLen(1)) Expect(msg.ToolCalls[0].Name).To(Equal("get_current_weather")) Expect(msg.ToolCalls[0].ID).To(Equal("call_abc123")) }) }) Context("StandardJSONTools Cohere format", func() { It("parses Cohere-style tool calls with custom keys", func() { tools := []peg.ToolDef{ { Name: "get_current_weather", Properties: map[string]peg.PropDef{ "location": {Type: "string"}, "unit": {Type: "string"}, }, }, } parser := peg.BuildChatPegParser(func(p *peg.ChatBuilder) peg.ParserID { toolCall := p.StandardJSONTools(peg.StandardJSONToolsOpts{ SectionStart: "<|START_ACTION|>[", SectionEnd: "]<|END_ACTION|>", Tools: tools, NameKey: "tool_name", ArgsKey: "parameters", GenCallIDKey: "tool_call_id", ParametersOrder: []string{"tool_call_id", "tool_name", "parameters"}, }) return p.Seq( p.Content(p.Until("<|START_ACTION|>")), p.Optional(p.Seq(p.Space(), toolCall)), p.End(), ) }) input := `Let me search for that.<|START_ACTION|>[{"tool_call_id": 0, "tool_name": "get_current_weather", "parameters": {"location": "NYC", "unit": "celsius"}}]<|END_ACTION|>` ctx := peg.NewParseContext(input, false) result := parser.Parse(ctx) Expect(result.Type).To(Equal(peg.Success)) mapper := &peg.ChatPegMapper{} mapper.FromAST(&ctx.Ast, &result) msg := mapper.Result Expect(msg.ToolCalls).To(HaveLen(1)) Expect(msg.ToolCalls[0].Name).To(Equal("get_current_weather")) Expect(msg.ToolCalls[0].ID).To(Equal("0")) }) }) Context("StandardJSONTools function-as-key format", func() { It("parses function name as JSON key", func() { tools := []peg.ToolDef{ { Name: "get_current_weather", Properties: map[string]peg.PropDef{ "location": {Type: "string"}, "unit": {Type: "string"}, }, }, } parser := peg.BuildChatPegParser(func(p *peg.ChatBuilder) peg.ParserID { toolCall := p.StandardJSONTools(peg.StandardJSONToolsOpts{ SectionStart: "[", SectionEnd: "]", Tools: tools, ArgsKey: "args", FunctionIsKey: true, CallIDKey: "id", }) return p.Seq( p.Content(p.Until("")), p.Optional(p.Seq(p.Space(), toolCall)), p.End(), ) }) input := `I'll call the weather function.[{"get_current_weather": {"id": "call-0001", "args": {"location": "NYC", "unit": "celsius"}}}]` ctx := peg.NewParseContext(input, false) result := parser.Parse(ctx) Expect(result.Type).To(Equal(peg.Success)) mapper := &peg.ChatPegMapper{} mapper.FromAST(&ctx.Ast, &result) msg := mapper.Result Expect(msg.ToolCalls).To(HaveLen(1)) Expect(msg.ToolCalls[0].Name).To(Equal("get_current_weather")) Expect(msg.ToolCalls[0].ID).To(Equal("call-0001")) }) }) Context("Tagged args with embedded quotes", func() { It("handles embedded double quotes in tagged parameters", func() { tools := []peg.ToolDef{ { Name: "edit", Properties: map[string]peg.PropDef{ "filename": {Type: "string"}, "oldString": {Type: "string"}, "newString": {Type: "string"}, }, }, } parser := peg.BuildChatPegParser(func(p *peg.ChatBuilder) peg.ParserID { toolCall := p.StandardConstructedTools( map[string]string{ "tool_call_start_marker": "", "tool_call_end_marker": "", "function_opener": "", "function_closer": "", "parameter_key_prefix": "", "parameter_closer": "", }, tools, false, true, ) return p.Seq(toolCall, p.Space(), p.End()) }) input := "\n" + "\n" + "\nfoo.cpp\n\n" + "def foo(arg = \"14\"):\n return arg + \"bar\"\n\n" + "def foo(arg = \"15\"):\n pass\n\n" + "\n" + "" ctx := peg.NewParseContext(input, false) result := parser.Parse(ctx) Expect(result.Type).To(Equal(peg.Success)) mapper := &peg.ChatPegMapper{} mapper.FromAST(&ctx.Ast, &result) msg := mapper.Result Expect(msg.ToolCalls).To(HaveLen(1)) Expect(msg.ToolCalls[0].Name).To(Equal("edit")) var parsed map[string]any err := json.Unmarshal([]byte(msg.ToolCalls[0].Arguments), &parsed) Expect(err).NotTo(HaveOccurred(), "arguments not valid JSON: %s", msg.ToolCalls[0].Arguments) Expect(parsed["filename"]).NotTo(BeNil()) }) }) })