feat(config): append tool result URLs to uncited profile answers

A research profile asks the model in its system prompt to cite its
sources, but a model can answer from a search result without naming a
single link, and the client never sees the server-side tool results that
held them.

evidence_check.sources appends one line with the distinct http(s) URLs
returned by the request's successful MCP tool calls (at most five, in
order of appearance) when the final answer names no URL itself. An answer
that already links something is left unchanged, and nothing is added when
no tool result holds a URL, so a link is never made up. The label is
configurable through evidence_check.sources_label (default "Sources") for
profiles that answer in another language.

It runs in the existing evidence gate, on the same streaming and
non-streaming paths as the number note.

Assisted-by: Claude Code:claude-opus-5-5
Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>

core/config/meta/registry.go

@@ -471,6 +471,20 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Component:   "input",
 			Order:       7,
 		},
+		"profile.evidence_check.sources": {
+			Section:     "alias",
+			Label:       "Profile: append tool sources",
+			Description: "Alias only. When the answer names no http(s) URL but the request's MCP tool results do, append one line with up to five of those URLs. URLs are only taken from tool results, never invented.",
+			Component:   "toggle",
+			Order:       8,
+		},
+		"profile.evidence_check.sources_label": {
+			Section:     "alias",
+			Label:       "Profile: source line label",
+			Description: "Alias only. Label of the line appended by profile.evidence_check.sources. Empty uses \"Sources\".",
+			Component:   "input",
+			Order:       9,
+		},
 
 		// --- Failover ---
 		"failover.targets": {

core/config/model_config.go

@@ -645,19 +645,29 @@ type ProfileConfig struct {
 	MCP MCPConfig `yaml:"mcp,omitempty" json:"mcp,omitempty"`
 	// ClientTools overrides the target's client_tools policy.
 	ClientTools string `yaml:"client_tools,omitempty" json:"client_tools,omitempty"`
-	// EvidenceCheck marks answers whose numbers have no evidence in the request.
+	// EvidenceCheck marks answers whose numbers have no evidence in the request
+	// and can cite the URLs the tools returned.
 	EvidenceCheck EvidenceCheckConfig `yaml:"evidence_check,omitempty" json:"evidence_check,omitempty"`
 }
 
 // DefaultEvidenceNote is appended when evidence_check.note is empty.
 const DefaultEvidenceNote = "_Note: this answer contains numbers that no tool returned in this turn — not verified._"
 
+// DefaultSourcesLabel heads the appended source line when
+// evidence_check.sources_label is empty.
+const DefaultSourcesLabel = "Sources"
+
 // @Description Output check of a profile alias
 type EvidenceCheckConfig struct {
 	// Numbers enables the check for numbers in the final answer.
 	Numbers bool `yaml:"numbers,omitempty" json:"numbers,omitempty"`
 	// Note replaces DefaultEvidenceNote.
 	Note string `yaml:"note,omitempty" json:"note,omitempty"`
+	// Sources appends the URLs returned by the request's tool calls to an
+	// answer that names no URL itself.
+	Sources bool `yaml:"sources,omitempty" json:"sources,omitempty"`
+	// SourcesLabel replaces DefaultSourcesLabel.
+	SourcesLabel string `yaml:"sources_label,omitempty" json:"sources_label,omitempty"`
 }
 
 // EvidenceCheck returns the evidence check a profile alias applied to this
@@ -672,6 +682,15 @@ func (e EvidenceCheckConfig) EvidenceNote() string {
 	return DefaultEvidenceNote
 }
 
+// SourcesHeading returns the configured label of the source line or the
+// default one.
+func (e EvidenceCheckConfig) SourcesHeading() string {
+	if e.SourcesLabel != "" {
+		return e.SourcesLabel
+	}
+	return DefaultSourcesLabel
+}
+
 // IsZero reports whether the block sets nothing.
 func (p ProfileConfig) IsZero() bool { return p == ProfileConfig{} }
 
@@ -2273,6 +2292,9 @@ func (c *ModelConfig) validateProfileFields() error {
 	if ec := c.Profile.EvidenceCheck; ec.Note != "" && !ec.Numbers {
 		return fmt.Errorf("profile.evidence_check.note has no effect without numbers: true")
 	}
+	if ec := c.Profile.EvidenceCheck; ec.SourcesLabel != "" && !ec.Sources {
+		return fmt.Errorf("profile.evidence_check.sources_label has no effect without sources: true")
+	}
 	return nil
 }
 

core/config/model_config_profile_alias_test.go

@@ -172,6 +172,30 @@ mcp:
 		Expect(base.EvidenceCheck().Numbers).To(BeFalse())
 	})
 
+	It("carries evidence_check.sources into the resolved config and nowhere else", func() {
+		dir := GinkgoT().TempDir()
+		Expect(os.WriteFile(filepath.Join(dir, "base-llm.yaml"), []byte(target), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(dir, "cited.yaml"), []byte("name: cited\nalias: base-llm\nprofile:\n  evidence_check:\n    sources: true\n    sources_label: Quellen\n"), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(dir, "default-label.yaml"), []byte("name: default-label\nalias: base-llm\nprofile:\n  evidence_check:\n    sources: true\n"), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(dir, "gpt-4.yaml"), []byte(plain), 0644)).To(Succeed())
+		cl = config.NewModelConfigLoader(dir)
+		Expect(cl.LoadModelConfigsFromPath(dir)).To(Succeed())
+
+		cited := resolve("cited")
+		Expect(cited.EvidenceCheck().Sources).To(BeTrue())
+		Expect(cited.EvidenceCheck().SourcesHeading()).To(Equal("Quellen"))
+		Expect(cited.EvidenceCheck().Numbers).To(BeFalse())
+
+		defaultLabel := resolve("default-label")
+		Expect(defaultLabel.EvidenceCheck().SourcesHeading()).To(Equal(config.DefaultSourcesLabel))
+
+		// A plain alias and the target itself stay off.
+		Expect(resolve("gpt-4").EvidenceCheck().Sources).To(BeFalse())
+		base, ok := cl.GetModelConfig("base-llm")
+		Expect(ok).To(BeTrue())
+		Expect(base.EvidenceCheck().Sources).To(BeFalse())
+	})
+
 	It("ignores a top-level evidence_check on a model, which only a profile can set", func() {
 		dir := GinkgoT().TempDir()
 		Expect(os.WriteFile(filepath.Join(dir, "base-llm.yaml"), []byte(target+"evidence_check:\n  numbers: true\n"), 0644)).To(Succeed())
@@ -213,6 +237,19 @@ mcp:
 			Expect(ok).To(BeTrue())
 		})
 
+		It("rejects an evidence_check sources_label without sources: true", func() {
+			cfg := config.ModelConfig{Name: "p", Alias: "base-llm"}
+			cfg.Profile.EvidenceCheck.SourcesLabel = "Quellen"
+			ok, err := cfg.Validate()
+			Expect(ok).To(BeFalse())
+			Expect(err).To(MatchError(ContainSubstring("profile.evidence_check.sources_label")))
+
+			cfg.Profile.EvidenceCheck.Sources = true
+			ok, err = cfg.Validate()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ok).To(BeTrue())
+		})
+
 		It("rejects a profile block on a model that is not an alias", func() {
 			cfg := config.ModelConfig{Name: "m", Backend: "llama-cpp"}
 			cfg.Profile.SystemPrompt = "persona"

core/http/endpoints/openai/chat.go

@@ -736,7 +736,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 								toolResult = fmt.Sprintf("Error: %v", toolErr)
 							} else {
 								// A failed call delivers no evidence for the answer.
-								evidence.markToolExecuted()
+								evidence.markToolExecuted(toolResult)
 							}
 							input.Messages = append(input.Messages, schema.Message{
 								Role:          "tool",
@@ -1186,7 +1186,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 								toolResult = fmt.Sprintf("Error: %v", toolErr)
 							} else {
 								// A failed call delivers no evidence for the answer.
-								evidence.markToolExecuted()
+								evidence.markToolExecuted(toolResult)
 							}
 							input.Messages = append(input.Messages, schema.Message{
 								Role:          "tool",

core/http/endpoints/openai/evidence.go

@@ -11,6 +11,37 @@ import (
 
 var numberToken = regexp.MustCompile(`\b\d+(?:[.,]\d+)?\b`)
 
+// urlToken matches an http(s) URL up to the first character that cannot be
+// part of one in tool output: whitespace, a quote, a closing bracket of
+// Markdown or JSON, or the backslash of a JSON escape.
+var urlToken = regexp.MustCompile(`https?://[^\s"'<>()\[\]{}\\]+`)
+
+// maxSourceURLs caps the appended source line; a search tool can return
+// dozens of links, and the first ones are the ones it ranked highest.
+const maxSourceURLs = 5
+
+// sourceURLs returns the distinct URLs of texts in the order they first
+// appear, at most maxSourceURLs. Punctuation that ends a sentence is not part
+// of the URL.
+func sourceURLs(texts []string) []string {
+	var urls []string
+	seen := map[string]struct{}{}
+	for _, text := range texts {
+		for _, token := range urlToken.FindAllString(text, -1) {
+			token = strings.TrimRight(token, ".,;:!?")
+			if _, ok := seen[token]; ok || strings.HasSuffix(token, "://") {
+				continue
+			}
+			seen[token] = struct{}{}
+			urls = append(urls, token)
+			if len(urls) == maxSourceURLs {
+				return urls
+			}
+		}
+	}
+	return urls
+}
+
 // numberKey makes "3,5" and "3.5" the same number, so an answer is not
 // flagged for writing a decimal with the other separator than its source.
 func numberKey(token string) string {
@@ -53,35 +84,63 @@ func hasUnsupportedNumber(answer string, evidence map[string]struct{}, toolExecu
 // evidenceGate carries the per-request state of a profile's evidence check.
 // A nil gate is disabled, so callers need no separate enabled check.
 type evidenceGate struct {
+	// evidence is nil when the number check is off.
 	note         string
 	evidence     map[string]struct{}
 	toolExecuted bool
+
+	sources      bool
+	sourcesLabel string
+	toolResults  []string
 }
 
 func newEvidenceGate(cfg *config.ModelConfig, messages []schema.Message) *evidenceGate {
 	check := cfg.EvidenceCheck()
-	if !check.Numbers {
+	if !check.Numbers && !check.Sources {
 		return nil
 	}
-	return &evidenceGate{note: check.EvidenceNote(), evidence: evidenceNumbers(cfg.SystemPrompt, messages)}
+	g := &evidenceGate{sources: check.Sources, sourcesLabel: check.SourcesHeading()}
+	if check.Numbers {
+		g.note = check.EvidenceNote()
+		g.evidence = evidenceNumbers(cfg.SystemPrompt, messages)
+	}
+	return g
 }
 
-func (g *evidenceGate) markToolExecuted() {
-	if g != nil {
-		g.toolExecuted = true
+// markToolExecuted records a successful server-side tool call. Its result is
+// the only place the source line may take a URL from.
+func (g *evidenceGate) markToolExecuted(result string) {
+	if g == nil {
+		return
+	}
+	g.toolExecuted = true
+	if g.sources {
+		g.toolResults = append(g.toolResults, result)
 	}
 }
 
 // suffix returns the text to append to answer, or "" when it needs none.
 func (g *evidenceGate) suffix(answer string) string {
-	if g == nil || !hasUnsupportedNumber(answer, g.evidence, g.toolExecuted) {
+	if g == nil {
 		return ""
 	}
-	xlog.Debug("Evidence check: answer has numbers without evidence in the request, appending note")
-	return "\n\n" + g.note
+	var suffix string
+	// An answer that links anything is left alone: it cites in its own way,
+	// and a second list would contradict or repeat it.
+	if g.sources && !urlToken.MatchString(answer) {
+		if urls := sourceURLs(g.toolResults); len(urls) > 0 {
+			xlog.Debug("Evidence check: answer names no URL, appending the tool results' URLs", "count", len(urls))
+			suffix += "\n\n" + g.sourcesLabel + ": " + strings.Join(urls, ", ")
+		}
+	}
+	if g.evidence != nil && hasUnsupportedNumber(answer, g.evidence, g.toolExecuted) {
+		xlog.Debug("Evidence check: answer has numbers without evidence in the request, appending note")
+		suffix += "\n\n" + g.note
+	}
+	return suffix
 }
 
-// apply appends the note to every final answer choice. Choices that hand tool
+// apply appends the suffix to every final answer choice. Choices that hand tool
 // calls to the client are not final answers and stay untouched.
 func (g *evidenceGate) apply(choices []schema.Choice) {
 	if g == nil {

core/http/endpoints/openai/evidence_sources_test.go

@@ -0,0 +1,29 @@
+package openai
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("sourceURLs", func() {
+	It("returns unique URLs in order of appearance, at most five", func() {
+		text := "x https://1.example/a, y (https://2.example/b) z https://1.example/a " +
+			"[l](https://3.example/c). \"https://4.example/d\" http://5.example/e; https://6.example/f"
+		Expect(sourceURLs([]string{text})).To(Equal([]string{
+			"https://1.example/a", "https://2.example/b", "https://3.example/c", "https://4.example/d", "http://5.example/e",
+		}))
+	})
+
+	It("keeps the order across several tool results", func() {
+		Expect(sourceURLs([]string{"https://b.example", "https://a.example https://b.example"})).
+			To(Equal([]string{"https://b.example", "https://a.example"}))
+	})
+
+	It("stops at a JSON escape", func() {
+		Expect(sourceURLs([]string{`"https://a.example/x\n"`})).To(Equal([]string{"https://a.example/x"}))
+	})
+
+	It("finds nothing in text without links", func() {
+		Expect(sourceURLs([]string{"ftp://a.example and www.b.example"})).To(BeEmpty())
+	})
+})

core/http/endpoints/openai/evidence_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
+	"github.com/modelcontextprotocol/go-sdk/mcp"
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
 	mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp"
@@ -270,6 +271,70 @@ var _ = Describe("ChatEndpoint evidence check", func() {
 		})
 	})
 
+	Context("sources", func() {
+		const withSources = "profile:\n  evidence_check:\n    sources: true\n    sources_label: Quellen\n"
+		const toolResult = `{"results":[{"title":"A","url":"https://a.example/oil"},{"title":"B","url":"https://b.example/test."}],"see":"https://a.example/oil"}`
+
+		toolThenAnswer := func(answer string) {
+			stubBackend(
+				backend.LLMResponse{ChatDeltas: []*pb.ChatDelta{{ToolCalls: []*pb.ToolCallDelta{{Id: "call-1", Name: "lookup", Arguments: "{}"}}}}},
+				backend.LLMResponse{Response: answer, ChatDeltas: []*pb.ChatDelta{{Content: answer}}},
+			)
+		}
+
+		for _, stream := range []bool{false, true} {
+			answer := func(rec *httptest.ResponseRecorder) string {
+				if stream {
+					return streamedAnswer(rec)
+				}
+				return answerOf(rec)
+			}
+
+			Context(map[bool]string{false: "non-streaming", true: "streaming"}[stream], func() {
+				It("appends the URLs the tools returned when the answer names none", func() {
+					fx := newMCPResultFixture(toolResult)
+					toolThenAnswer("A good oil, according to two tests.")
+					Expect(answer(run(resolve(withSources, fx.url), stream, "Is this a good olive oil?"))).
+						To(HaveSuffix("A good oil, according to two tests.\n\nQuellen: https://a.example/oil, https://b.example/test"))
+				})
+
+				It("leaves an answer that names a URL unchanged", func() {
+					fx := newMCPResultFixture(toolResult)
+					toolThenAnswer("A good oil, see https://a.example/oil.")
+					Expect(answer(run(resolve(withSources, fx.url), stream, "Is this a good olive oil?"))).
+						To(HaveSuffix("A good oil, see https://a.example/oil."))
+				})
+
+				It("adds nothing when no tool result holds a URL", func() {
+					fx := newMCPResultFixture("no links here")
+					toolThenAnswer("A good oil.")
+					Expect(answer(run(resolve(withSources, fx.url), stream, "Is this a good olive oil?"))).
+						To(HaveSuffix("A good oil."))
+				})
+
+				It("adds nothing without the switch", func() {
+					fx := newMCPResultFixture(toolResult)
+					toolThenAnswer("A good oil.")
+					Expect(answer(run(resolve(withoutCheck, fx.url), stream, "Is this a good olive oil?"))).
+						To(HaveSuffix("A good oil."))
+				})
+			})
+		}
+
+		It("takes no URL from the request, only from tool results", func() {
+			stubBackend(backend.LLMResponse{Response: "A good oil."})
+			Expect(answerOf(run(resolve(withSources, ""), false, "Is https://shop.example/oil a good olive oil?"))).
+				To(Equal("A good oil."))
+		})
+
+		It("uses the English label by default", func() {
+			fx := newMCPResultFixture(toolResult)
+			toolThenAnswer("A good oil.")
+			Expect(answerOf(run(resolve("profile:\n  evidence_check:\n    sources: true\n", fx.url), false, "Is this a good olive oil?"))).
+				To(Equal("A good oil.\n\nSources: https://a.example/oil, https://b.example/test"))
+		})
+	})
+
 	Context("streaming", func() {
 		It("sends the note as a last content delta", func() {
 			stubBackend(backend.LLMResponse{Response: "There are 0 open invoices."})
@@ -313,3 +378,22 @@ var _ = Describe("ChatEndpoint evidence check", func() {
 		})
 	})
 })
+
+// newMCPResultFixture is a remote MCP server whose one tool, lookup, returns
+// result as text.
+func newMCPResultFixture(result string) *mcpAutoFixture {
+	fx := &mcpAutoFixture{}
+	srv := mcp.NewServer(&mcp.Implementation{Name: "fixture", Version: "v0.0.1"}, nil)
+	srv.AddTool(&mcp.Tool{
+		Name:        "lookup",
+		Description: "fixture tool",
+		InputSchema: json.RawMessage(`{"type":"object","properties":{}}`),
+	}, func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil
+	})
+	handler := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return srv }, nil)
+	server := httptest.NewServer(handler)
+	DeferCleanup(server.Close)
+	fx.url = server.URL
+	return fx
+}

docs/content/features/model-aliases.md

@@ -56,6 +56,8 @@ top of the target's config for requests to the alias:
 | `profile.client_tools: drop` | Discards tools and `tool_choice` sent by the client. |
 | `profile.evidence_check.numbers` | Marks answers that contain numbers without evidence in the request. See [Evidence check](#evidence-check). |
 | `profile.evidence_check.note` | Text appended by the evidence check. Defaults to an English note. |
+| `profile.evidence_check.sources` | Appends the URLs the request's MCP tools returned to an answer that names no URL. See [Source line](#source-line). |
+| `profile.evidence_check.sources_label` | Label of the source line. Defaults to `Sources`. |
 
 ```yaml
 name: researcher
@@ -126,6 +128,46 @@ unless `numbers: true` is set, and `note` without `numbers: true` is rejected.
 It flags, it does not prove: a number the client happened to mention is
 accepted even if the answer uses it wrongly.
 
+### Source line
+
+A research profile usually asks the model in its system prompt to cite its
+sources. A model can still answer from a search result without naming a single
+link, and the client never sees the tool results that held them. With
+`sources: true` LocalAI adds the links at the output:
+
+```yaml
+name: researcher
+alias: my-llama-3
+profile:
+  mcp:
+    auto: true
+    remote: |
+      {"mcpServers": {"search": {"url": "http://search.internal/mcp"}}}
+  evidence_check:
+    sources: true
+    # optional, defaults to "Sources"
+    sources_label: Quellen
+```
+
+Like the number check it acts on `/v1/chat/completions`. The rule is
+deterministic and applies to each request on its own:
+
+1. Only the results of MCP tool calls that succeeded on the server during this
+   request are read. The client's messages and failed calls are not a source.
+2. If the final answer already contains an `http://` or `https://` URL, it is
+   left unchanged.
+3. Otherwise the distinct URLs of those tool results are collected in the order
+   they first appear, at most five, and appended after a blank line as one line:
+   `Quellen: https://a.example/test, https://b.example/oil`. Punctuation that
+   ends a sentence is not taken as part of a URL.
+4. If no tool result contains a URL, nothing is appended. No URL is ever made
+   up: every link on the line was returned by a tool.
+
+In a streamed response the line arrives as one additional content chunk before
+the final chunk, like the number note. Both checks can be enabled together; the
+number note follows the source line. `sources_label` without `sources: true` is
+rejected.
+
 ## Managing aliases
 
 You can create, swap, and remove aliases from any of the management surfaces.