feat(config): mark links a profile answer names without retrieving them

A model can cite a plausible deep link it never opened, written from
memory. The client cannot tell it from a link a search tool returned,
because server-side MCP tool results never reach it.

evidence_check.urls compares every http(s) link of the final answer,
bare or inside a Markdown link, with the URLs of the request (system
prompt, system, user and tool messages) and of the request's successful
MCP tool results. Links are compared without scheme, fragment, "www."
and trailing slash; a link counts as retrieved when it equals an evidence
URL or is a shorter form of one at a path or query boundary. Unbacked
links (at most five) are listed in an appended note; urls_note replaces
the default note, with {urls} as the placeholder for the list.

Assistant messages are not evidence, so an invented link is marked again
when the client sends it back on the next turn.

Assisted-by: Claude Code:claude-opus-5-5
Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>

core/config/meta/registry.go

@@ -485,12 +485,26 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Component:   "input",
 			Order:       9,
 		},
+		"profile.evidence_check.urls": {
+			Section:     "alias",
+			Label:       "Profile: mark links not retrieved",
+			Description: "Alias only. When the answer names an http(s) link found neither in the request (system prompt, system, user and tool messages) nor in a successful MCP tool result of the request, append a note listing up to five such links.",
+			Component:   "toggle",
+			Order:       10,
+		},
+		"profile.evidence_check.urls_note": {
+			Section:     "alias",
+			Label:       "Profile: links-not-retrieved note",
+			Description: "Alias only. Text appended by profile.evidence_check.urls; {urls} is replaced by the links. Empty uses the built-in English note.",
+			Component:   "input",
+			Order:       11,
+		},
 		"profile.require_tool": {
 			Section:     "alias",
 			Label:       "Profile: require a tool call first",
 			Description: "Alias only. The first model call of a request with server-side MCP tools is made with tool_choice \"required\"; the follow-up calls of the tool loop use the client's tool_choice again.",
 			Component:   "toggle",
-			Order:       10,
+			Order:       12,
 		},
 
 		// --- Failover ---

core/config/model_config.go

@@ -646,8 +646,8 @@ type ProfileConfig struct {
 	MCP MCPConfig `yaml:"mcp,omitempty" json:"mcp,omitempty"`
 	// ClientTools overrides the target's client_tools policy.
 	ClientTools string `yaml:"client_tools,omitempty" json:"client_tools,omitempty"`
-	// EvidenceCheck marks answers whose numbers have no evidence in the request
-	// and can cite the URLs the tools returned.
+	// EvidenceCheck marks answers whose numbers or links have no evidence in
+	// the request and can cite the URLs the tools returned.
 	EvidenceCheck EvidenceCheckConfig `yaml:"evidence_check,omitempty" json:"evidence_check,omitempty"`
 	// RequireTool makes the first model call of a request with server-side
 	// MCP tools call a tool (tool_choice "required"). Follow-up calls of the
@@ -662,6 +662,13 @@ const DefaultEvidenceNote = "_Note: this answer contains numbers that no tool re
 // evidence_check.sources_label is empty.
 const DefaultSourcesLabel = "Sources"
 
+// URLsPlaceholder in evidence_check.urls_note is replaced by the links the
+// answer names without evidence.
+const URLsPlaceholder = "{urls}"
+
+// DefaultURLsNote is appended when evidence_check.urls_note is empty.
+const DefaultURLsNote = "_Note: this answer names links that no tool returned in this turn — not retrieved: " + URLsPlaceholder + "_"
+
 // @Description Output check of a profile alias
 type EvidenceCheckConfig struct {
 	// Numbers enables the check for numbers in the final answer.
@@ -673,6 +680,11 @@ type EvidenceCheckConfig struct {
 	Sources bool `yaml:"sources,omitempty" json:"sources,omitempty"`
 	// SourcesLabel replaces DefaultSourcesLabel.
 	SourcesLabel string `yaml:"sources_label,omitempty" json:"sources_label,omitempty"`
+	// URLs marks answers that name an http(s) link found neither in the
+	// request nor in a successful tool result of the request.
+	URLs bool `yaml:"urls,omitempty" json:"urls,omitempty"`
+	// URLsNote replaces DefaultURLsNote.
+	URLsNote string `yaml:"urls_note,omitempty" json:"urls_note,omitempty"`
 }
 
 // EvidenceCheck returns the evidence check a profile alias applied to this
@@ -696,6 +708,21 @@ func (e EvidenceCheckConfig) SourcesHeading() string {
 	return DefaultSourcesLabel
 }
 
+// UnverifiedURLsNote returns the note for the given links: the configured or
+// default note with URLsPlaceholder replaced, or with the links appended when
+// the note has no placeholder.
+func (e EvidenceCheckConfig) UnverifiedURLsNote(urls []string) string {
+	note := e.URLsNote
+	if note == "" {
+		note = DefaultURLsNote
+	}
+	list := strings.Join(urls, ", ")
+	if strings.Contains(note, URLsPlaceholder) {
+		return strings.ReplaceAll(note, URLsPlaceholder, list)
+	}
+	return note + " " + list
+}
+
 // RequiresTool reports whether a profile alias asked for a forced tool call on
 // the first model call of a request; it is false for every other config.
 func (c ModelConfig) RequiresTool() bool { return c.requireTool }
@@ -2305,6 +2332,9 @@ func (c *ModelConfig) validateProfileFields() error {
 	if ec := c.Profile.EvidenceCheck; ec.SourcesLabel != "" && !ec.Sources {
 		return fmt.Errorf("profile.evidence_check.sources_label has no effect without sources: true")
 	}
+	if ec := c.Profile.EvidenceCheck; ec.URLsNote != "" && !ec.URLs {
+		return fmt.Errorf("profile.evidence_check.urls_note has no effect without urls: true")
+	}
 	return nil
 }
 

core/config/model_config_profile_alias_test.go

@@ -3,6 +3,7 @@ package config_test
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -172,6 +173,33 @@ mcp:
 		Expect(base.EvidenceCheck().Numbers).To(BeFalse())
 	})
 
+	It("carries evidence_check.urls into the resolved config and nowhere else", func() {
+		dir := GinkgoT().TempDir()
+		Expect(os.WriteFile(filepath.Join(dir, "base-llm.yaml"), []byte(target), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(dir, "linked.yaml"), []byte("name: linked\nalias: base-llm\nprofile:\n  evidence_check:\n    urls: true\n    urls_note: \"nicht abgerufen: {urls}\"\n"), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(dir, "default-note.yaml"), []byte("name: default-note\nalias: base-llm\nprofile:\n  evidence_check:\n    urls: true\n"), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(dir, "gpt-4.yaml"), []byte(plain), 0644)).To(Succeed())
+		cl = config.NewModelConfigLoader(dir)
+		Expect(cl.LoadModelConfigsFromPath(dir)).To(Succeed())
+
+		linked := resolve("linked")
+		Expect(linked.EvidenceCheck().URLs).To(BeTrue())
+		Expect(linked.EvidenceCheck().UnverifiedURLsNote([]string{"https://a.example", "https://b.example"})).
+			To(Equal("nicht abgerufen: https://a.example, https://b.example"))
+		Expect(resolve("default-note").EvidenceCheck().UnverifiedURLsNote([]string{"https://a.example"})).
+			To(Equal(strings.ReplaceAll(config.DefaultURLsNote, config.URLsPlaceholder, "https://a.example")))
+
+		Expect(resolve("gpt-4").EvidenceCheck().URLs).To(BeFalse())
+		base, ok := cl.GetModelConfig("base-llm")
+		Expect(ok).To(BeTrue())
+		Expect(base.EvidenceCheck().URLs).To(BeFalse())
+	})
+
+	It("appends the links to a urls_note without placeholder", func() {
+		Expect(config.EvidenceCheckConfig{URLs: true, URLsNote: "Not retrieved:"}.UnverifiedURLsNote([]string{"https://a.example"})).
+			To(Equal("Not retrieved: https://a.example"))
+	})
+
 	It("carries evidence_check.sources and require_tool into the resolved config and nowhere else", func() {
 		dir := GinkgoT().TempDir()
 		Expect(os.WriteFile(filepath.Join(dir, "base-llm.yaml"), []byte(target+"require_tool: true\n"), 0644)).To(Succeed())
@@ -253,6 +281,19 @@ mcp:
 			Expect(ok).To(BeTrue())
 		})
 
+		It("rejects an evidence_check urls_note without urls: true", func() {
+			cfg := config.ModelConfig{Name: "p", Alias: "base-llm"}
+			cfg.Profile.EvidenceCheck.URLsNote = "nicht abgerufen: {urls}"
+			ok, err := cfg.Validate()
+			Expect(ok).To(BeFalse())
+			Expect(err).To(MatchError(ContainSubstring("profile.evidence_check.urls_note")))
+
+			cfg.Profile.EvidenceCheck.URLs = true
+			ok, err = cfg.Validate()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ok).To(BeTrue())
+		})
+
 		It("accepts require_tool on a profile alias only", func() {
 			cfg := config.ModelConfig{Name: "p", Alias: "base-llm"}
 			cfg.Profile.RequireTool = true

core/http/endpoints/openai/evidence.go

@@ -21,28 +21,43 @@ var urlToken = regexp.MustCompile(`https?://[^\s"'<>\[\]{}\\]+`)
 // dozens of links, and the first ones are the ones it ranked highest.
 const maxSourceURLs = 5
 
+// extractURLs returns every http(s) URL of text in order, duplicates
+// included. Punctuation that ends a sentence and unmatched closing parentheses
+// of Markdown links are not part of the URL.
+func extractURLs(text string) []string {
+	// The MCP tool call path JSON-encodes multiple TextContent results.
+	// Decode that envelope so escaped query separators remain intact.
+	var parts []string
+	var single string
+	if json.Unmarshal([]byte(text), &parts) == nil {
+		text = strings.Join(parts, "\n")
+	} else if json.Unmarshal([]byte(text), &single) == nil {
+		text = single
+	}
+	// Some JSON encoders escape slashes inside string values.
+	text = strings.ReplaceAll(text, `\/`, "/")
+	var urls []string
+	for _, token := range urlToken.FindAllString(text, -1) {
+		token = strings.TrimRight(token, ".,;:!?")
+		for strings.HasSuffix(token, ")") && strings.Count(token, ")") > strings.Count(token, "(") {
+			token = strings.TrimRight(strings.TrimSuffix(token, ")"), ".,;:!?")
+		}
+		if strings.HasSuffix(token, "://") {
+			continue
+		}
+		urls = append(urls, token)
+	}
+	return urls
+}
+
 // sourceURLs returns the distinct URLs of texts in the order they first
-// appear, at most maxSourceURLs. Punctuation that ends a sentence is not part
-// of the URL.
+// appear, at most maxSourceURLs.
 func sourceURLs(texts []string) []string {
 	var urls []string
 	seen := map[string]struct{}{}
 	for _, text := range texts {
-		// The MCP tool call path JSON-encodes multiple TextContent results.
-		// Decode that envelope so escaped query separators remain intact.
-		var parts []string
-		var single string
-		if json.Unmarshal([]byte(text), &parts) == nil {
-			text = strings.Join(parts, "\n")
-		} else if json.Unmarshal([]byte(text), &single) == nil {
-			text = single
-		}
-		for _, token := range urlToken.FindAllString(text, -1) {
-			token = strings.TrimRight(token, ".,;:!?")
-			for strings.HasSuffix(token, ")") && strings.Count(token, ")") > strings.Count(token, "(") {
-				token = strings.TrimRight(strings.TrimSuffix(token, ")"), ".,;:!?")
-			}
-			if _, ok := seen[token]; ok || strings.HasSuffix(token, "://") {
+		for _, token := range extractURLs(text) {
+			if _, ok := seen[token]; ok {
 				continue
 			}
 			seen[token] = struct{}{}
@@ -55,6 +70,100 @@ func sourceURLs(texts []string) []string {
 	return urls
 }
 
+// urlKey reduces a URL to what decides whether two links name the same page:
+// no scheme, no fragment, lower-case host without "www.", no trailing slash.
+// The path and query stay case-sensitive.
+func urlKey(raw string) string {
+	u := raw
+	if i := strings.Index(u, "://"); i >= 0 {
+		u = u[i+3:]
+	}
+	if i := strings.IndexByte(u, '#'); i >= 0 {
+		u = u[:i]
+	}
+	host, rest := u, ""
+	if i := strings.IndexAny(u, "/?"); i >= 0 {
+		host, rest = u[:i], u[i:]
+	}
+	host = strings.TrimPrefix(strings.ToLower(host), "www.")
+	path, query, hasQuery := strings.Cut(rest, "?")
+	key := host + strings.TrimRight(path, "/")
+	if hasQuery && query != "" {
+		key += "?" + query
+	}
+	return key
+}
+
+// urlEvidenced reports whether the link key is backed by one of the evidence
+// keys: the same page, or a shorter form of a link the evidence holds (the
+// site or a parent path of a page that was retrieved). A deeper or different
+// path than any evidence is not backed: an invented deep link on a known site
+// is exactly what the check is for.
+func urlEvidenced(key string, evidence map[string]struct{}) bool {
+	if _, ok := evidence[key]; ok {
+		return true
+	}
+	for e := range evidence {
+		if strings.HasPrefix(e, key) && len(e) > len(key) && (e[len(key)] == '/' || e[len(key)] == '?') {
+			return true
+		}
+	}
+	return false
+}
+
+// evidenceURLs collects the link keys of the request as received: the
+// profile's system prompt and the system, user and tool messages. Assistant
+// messages are left out, so a link the model invented on an earlier turn does
+// not become evidence once the client sends it back.
+func evidenceURLs(systemPrompt string, messages []schema.Message) map[string]struct{} {
+	keys := map[string]struct{}{}
+	add := func(text string) {
+		for _, u := range extractURLs(text) {
+			keys[urlKey(u)] = struct{}{}
+		}
+	}
+	add(systemPrompt)
+	for _, m := range messages {
+		if m.Role == "assistant" {
+			continue
+		}
+		add(messageText(m))
+	}
+	return keys
+}
+
+// unverifiedURLs returns the distinct links of answer, at most
+// maxSourceURLs, that are backed neither by the request nor by toolResults.
+func unverifiedURLs(answer string, request map[string]struct{}, toolResults []string) []string {
+	links := extractURLs(answer)
+	if len(links) == 0 {
+		return nil
+	}
+	evidence := make(map[string]struct{}, len(request))
+	for k := range request {
+		evidence[k] = struct{}{}
+	}
+	for _, r := range toolResults {
+		for _, u := range extractURLs(r) {
+			evidence[urlKey(u)] = struct{}{}
+		}
+	}
+	var out []string
+	seen := map[string]struct{}{}
+	for _, u := range links {
+		key := urlKey(u)
+		if _, ok := seen[key]; ok || urlEvidenced(key, evidence) {
+			continue
+		}
+		seen[key] = struct{}{}
+		out = append(out, u)
+		if len(out) == maxSourceURLs {
+			break
+		}
+	}
+	return out
+}
+
 // numberKey makes "3,5" and "3.5" the same number, so an answer is not
 // flagged for writing a decimal with the other separator than its source.
 func numberKey(token string) string {
@@ -105,11 +214,15 @@ type evidenceGate struct {
 	sources      bool
 	sourcesLabel string
 	toolResults  []string
+
+	// requestURLs is nil when the link check is off.
+	requestURLs map[string]struct{}
+	urlsNote    func([]string) string
 }
 
 func newEvidenceGate(cfg *config.ModelConfig, messages []schema.Message) *evidenceGate {
 	check := cfg.EvidenceCheck()
-	if !check.Numbers && !check.Sources {
+	if !check.Numbers && !check.Sources && !check.URLs {
 		return nil
 	}
 	g := &evidenceGate{sources: check.Sources, sourcesLabel: check.SourcesHeading()}
@@ -117,17 +230,22 @@ func newEvidenceGate(cfg *config.ModelConfig, messages []schema.Message) *eviden
 		g.note = check.EvidenceNote()
 		g.evidence = evidenceNumbers(cfg.SystemPrompt, messages)
 	}
+	if check.URLs {
+		g.requestURLs = evidenceURLs(cfg.SystemPrompt, messages)
+		g.urlsNote = check.UnverifiedURLsNote
+	}
 	return g
 }
 
 // markToolExecuted records a successful server-side tool call. Its result is
-// the only place the source line may take a URL from.
+// the only place the source line may take a URL from, and evidence for the
+// links the answer names.
 func (g *evidenceGate) markToolExecuted(result string) {
 	if g == nil {
 		return
 	}
 	g.toolExecuted = true
-	if g.sources {
+	if g.sources || g.requestURLs != nil {
 		g.toolResults = append(g.toolResults, result)
 	}
 }
@@ -146,6 +264,12 @@ func (g *evidenceGate) suffix(answer string) string {
 			suffix += "\n\n" + g.sourcesLabel + ": " + strings.Join(urls, ", ")
 		}
 	}
+	if g.requestURLs != nil {
+		if urls := unverifiedURLs(answer, g.requestURLs, g.toolResults); len(urls) > 0 {
+			xlog.Debug("Evidence check: answer names links no tool returned, appending note", "count", len(urls))
+			suffix += "\n\n" + g.urlsNote(urls)
+		}
+	}
 	if g.evidence != nil && hasUnsupportedNumber(answer, g.evidence, g.toolExecuted) {
 		xlog.Debug("Evidence check: answer has numbers without evidence in the request, appending note")
 		suffix += "\n\n" + g.note

core/http/endpoints/openai/evidence_test.go

@@ -335,6 +335,83 @@ var _ = Describe("ChatEndpoint evidence check", func() {
 		})
 	})
 
+	Context("urls", func() {
+		const withURLs = "profile:\n  evidence_check:\n    urls: true\n    urls_note: \"Nicht abgerufen: {urls}\"\n"
+		const toolResult = `{"results":[{"title":"A","url":"https://a.example/oil"}]}`
+
+		toolThenAnswer := func(answer string) {
+			stubBackend(
+				backend.LLMResponse{ChatDeltas: []*pb.ChatDelta{{ToolCalls: []*pb.ToolCallDelta{{Id: "call-1", Name: "lookup", Arguments: "{}"}}}}},
+				backend.LLMResponse{Response: answer, ChatDeltas: []*pb.ChatDelta{{Content: answer}}},
+			)
+		}
+
+		for _, stream := range []bool{false, true} {
+			answer := func(rec *httptest.ResponseRecorder) string {
+				if stream {
+					return streamedAnswer(rec)
+				}
+				return answerOf(rec)
+			}
+
+			Context(map[bool]string{false: "non-streaming", true: "streaming"}[stream], func() {
+				It("marks a link no tool returned", func() {
+					stubBackend(backend.LLMResponse{Response: "Per [Statista](https://www.statista.com/statistics/1/x), it grows."})
+					Expect(answer(run(resolve(withURLs, ""), stream, "Does the market grow?"))).
+						To(Equal("Per [Statista](https://www.statista.com/statistics/1/x), it grows.\n\nNicht abgerufen: https://www.statista.com/statistics/1/x"))
+				})
+
+				It("marks an invented link next to a retrieved one", func() {
+					fx := newMCPResultFixture(toolResult)
+					toolThenAnswer("See https://a.example/oil and https://gartner.example/report.")
+					Expect(answer(run(resolve(withURLs, fx.url), stream, "Is this a good olive oil?"))).
+						To(HaveSuffix("See https://a.example/oil and https://gartner.example/report.\n\nNicht abgerufen: https://gartner.example/report"))
+				})
+
+				It("leaves a link from a tool result unmarked", func() {
+					fx := newMCPResultFixture(toolResult)
+					toolThenAnswer("See https://a.example/oil/.")
+					Expect(answer(run(resolve(withURLs, fx.url), stream, "Is this a good olive oil?"))).
+						To(HaveSuffix("See https://a.example/oil/."))
+				})
+
+				It("leaves a link from the user message unmarked", func() {
+					stubBackend(backend.LLMResponse{Response: "https://shop.example/oil sells a good oil."})
+					Expect(answer(run(resolve(withURLs, ""), stream, "Is https://shop.example/oil a good olive oil?"))).
+						To(Equal("https://shop.example/oil sells a good oil."))
+				})
+
+				It("changes nothing without urls: true", func() {
+					stubBackend(backend.LLMResponse{Response: "See https://www.statista.com/statistics/1/x."})
+					Expect(answer(run(resolve(withoutCheck, ""), stream, "Does the market grow?"))).
+						To(Equal("See https://www.statista.com/statistics/1/x."))
+				})
+			})
+		}
+
+		It("marks a link when the MCP tool call failed", func() {
+			fx := newMCPResultFixture(toolResult)
+			stubBackend(
+				backend.LLMResponse{ChatDeltas: []*pb.ChatDelta{{ToolCalls: []*pb.ToolCallDelta{{Id: "call-1", Name: "lookup", Arguments: "{not json"}}}}},
+				backend.LLMResponse{Response: "See https://a.example/oil.", ChatDeltas: []*pb.ChatDelta{{Content: "See https://a.example/oil."}}},
+			)
+			Expect(answerOf(run(resolve(withURLs, fx.url), false, "Is this a good olive oil?"))).
+				To(Equal("See https://a.example/oil.\n\nNicht abgerufen: https://a.example/oil"))
+		})
+
+		It("uses the English note by default", func() {
+			stubBackend(backend.LLMResponse{Response: "See https://statista.example/x."})
+			Expect(answerOf(run(resolve("profile:\n  evidence_check:\n    urls: true\n", ""), false, "Market?"))).
+				To(Equal("See https://statista.example/x.\n\n" + strings.ReplaceAll(config.DefaultURLsNote, config.URLsPlaceholder, "https://statista.example/x")))
+		})
+
+		It("keeps the number check unchanged when both are on", func() {
+			stubBackend(backend.LLMResponse{Response: "There are 3 reports, see https://statista.example/x."})
+			Expect(answerOf(run(resolve(withURLs+"    numbers: true\n    note: "+note+"\n", ""), false, "How many reports?"))).
+				To(Equal("There are 3 reports, see https://statista.example/x.\n\nNicht abgerufen: https://statista.example/x\n\n" + note))
+		})
+	})
+
 	Context("streaming", func() {
 		It("sends the note as a last content delta", func() {
 			stubBackend(backend.LLMResponse{Response: "There are 0 open invoices."})

core/http/endpoints/openai/evidence_urls_test.go

@@ -0,0 +1,101 @@
+package openai
+
+import (
+	"github.com/mudler/LocalAI/core/schema"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("urlKey", func() {
+	DescribeTable("normalises links that name the same page to one key",
+		func(raw, want string) { Expect(urlKey(raw)).To(Equal(want)) },
+		Entry("drops the scheme", "https://example.com/a", "example.com/a"),
+		Entry("treats http and https alike", "http://example.com/a", "example.com/a"),
+		Entry("drops www. and lower-cases the host", "https://WWW.Example.com/A", "example.com/A"),
+		Entry("drops a trailing slash", "https://example.com/a/", "example.com/a"),
+		Entry("drops the fragment", "https://example.com/a#section", "example.com/a"),
+		Entry("keeps the query", "https://example.com/a/?q=1#x", "example.com/a?q=1"),
+		Entry("reduces a bare host", "https://example.com/", "example.com"),
+	)
+})
+
+var _ = Describe("unverifiedURLs", func() {
+	request := func(texts ...string) map[string]struct{} {
+		return evidenceURLs("", userMessages(texts...))
+	}
+
+	DescribeTable("lists the links of the answer that no evidence backs",
+		func(answer string, req map[string]struct{}, toolResults []string, want []string) {
+			got := unverifiedURLs(answer, req, toolResults)
+			if want == nil {
+				Expect(got).To(BeEmpty())
+			} else {
+				Expect(got).To(Equal(want))
+			}
+		},
+		Entry("an invented link",
+			"See https://www.statista.com/statistics/123/market for details.",
+			request("How big is the market?"), nil,
+			[]string{"https://www.statista.com/statistics/123/market"}),
+		Entry("a link a tool returned",
+			"See https://a.example/oil.",
+			request("Is this oil good?"), []string{`{"url":"https://a.example/oil"}`},
+			nil),
+		Entry("a link from the user message",
+			"The page https://shop.example/oil says so.",
+			request("Is https://shop.example/oil a good olive oil?"), nil,
+			nil),
+		Entry("a Markdown link that was not retrieved",
+			"According to [Gartner](https://www.gartner.com/en/newsroom/press-releases/2026).",
+			request("What do analysts say?"), []string{"https://a.example/oil"},
+			[]string{"https://www.gartner.com/en/newsroom/press-releases/2026"}),
+		Entry("a Markdown link a tool returned",
+			"According to [the test](https://a.example/oil).",
+			request("Is it good?"), []string{"Result: https://a.example/oil"},
+			nil),
+		Entry("trailing punctuation, slash, fragment and www. do not matter",
+			"Read https://www.a.example/oil/#top, then (https://b.example/test).",
+			request("?"), []string{"https://a.example/oil and https://b.example/test/"},
+			nil),
+		Entry("the site of a retrieved page counts as retrieved",
+			"Source: https://a.example",
+			request("?"), []string{"https://a.example/reports/2026/oil.pdf"},
+			nil),
+		Entry("a deeper path than the retrieved page does not",
+			"Source: https://a.example/reports/2026/other.pdf",
+			request("?"), []string{"https://a.example/reports/2026/oil.pdf"},
+			[]string{"https://a.example/reports/2026/other.pdf"}),
+		Entry("a host that only starts like a retrieved one does not",
+			"Source: https://a.ex",
+			request("?"), []string{"https://a.example/oil"},
+			[]string{"https://a.ex"}),
+		Entry("a link escaped by a JSON encoder in the tool result",
+			"Source: https://a.example/oil",
+			request("?"), []string{`{"url":"https:\/\/a.example\/oil"}`},
+			nil),
+		Entry("each unbacked link once, at most five",
+			"https://1.example https://2.example https://1.example/ https://3.example https://4.example https://5.example https://6.example",
+			request("?"), nil,
+			[]string{"https://1.example", "https://2.example", "https://3.example", "https://4.example", "https://5.example"}),
+		Entry("an answer without links",
+			"No link here.", request("?"), nil, nil),
+	)
+
+	It("does not take a link from an earlier assistant turn as evidence", func() {
+		msgs := []schema.Message{
+			{Role: "user", Content: "What do analysts say?", StringContent: "What do analysts say?"},
+			{Role: "assistant", Content: "See https://gartner.example/x", StringContent: "See https://gartner.example/x"},
+			{Role: "user", Content: "And?", StringContent: "And?"},
+		}
+		Expect(unverifiedURLs("Again https://gartner.example/x", evidenceURLs("", msgs), nil)).
+			To(Equal([]string{"https://gartner.example/x"}))
+	})
+
+	It("takes the system prompt and client tool messages as evidence", func() {
+		msgs := []schema.Message{
+			{Role: "tool", Content: "https://tool.example/r", StringContent: "https://tool.example/r"},
+		}
+		Expect(unverifiedURLs("https://sys.example/a and https://tool.example/r", evidenceURLs("Use https://sys.example/a", msgs), nil)).
+			To(BeEmpty())
+	})
+})

docs/content/features/model-aliases.md

@@ -58,6 +58,8 @@ top of the target's config for requests to the alias:
 | `profile.evidence_check.note` | Text appended by the evidence check. Defaults to an English note. |
 | `profile.evidence_check.sources` | Appends the URLs the request's MCP tools returned to an answer that names no URL. See [Source line](#source-line). |
 | `profile.evidence_check.sources_label` | Label of the source line. Defaults to `Sources`. |
+| `profile.evidence_check.urls` | Marks answers that name links neither the request nor a tool returned. See [Links not retrieved](#links-not-retrieved). |
+| `profile.evidence_check.urls_note` | Text appended by the link check; `{urls}` is replaced by the links. Defaults to an English note. |
 | `profile.require_tool` | Makes the first model call of a request call one of the profile's MCP tools. See [Requiring a tool call](#requiring-a-tool-call). |
 
 ```yaml
@@ -170,6 +172,53 @@ the final chunk, like the number note. Both checks can be enabled together; the
 number note follows the source line. `sources_label` without `sources: true` is
 rejected.
 
+### Links not retrieved
+
+A model can also name a link it never opened: a plausible deep link on a known
+site, written from memory. The client cannot tell such a link from one a search
+tool returned, because it never sees the tool results. With `urls: true` LocalAI
+marks it at the output:
+
+```yaml
+name: researcher
+alias: my-llama-3
+profile:
+  mcp:
+    auto: true
+    remote: |
+      {"mcpServers": {"search": {"url": "http://search.internal/mcp"}}}
+  evidence_check:
+    urls: true
+    # optional; {urls} is replaced by the links, defaults to the note below
+    urls_note: "_Note: this answer names links that no tool returned in this turn — not retrieved: {urls}_"
+```
+
+Like the other checks it acts on `/v1/chat/completions`, streamed or not. The
+rule is deterministic and applies to each request on its own:
+
+1. Every `http://` or `https://` URL in the final answer is collected, bare or
+   inside a Markdown link. Punctuation that ends a sentence is not part of it.
+2. Evidence is every URL in the request as received — the profile's system
+   prompt and the content of every system, user and tool message — and in the
+   results of the MCP tool calls that succeeded on the server during this
+   request. Assistant messages are not evidence, so a link the model invented on
+   an earlier turn is marked again when the client sends it back. Failed tool
+   calls deliver no evidence.
+3. Links are compared without scheme, fragment, a leading `www.` and a trailing
+   slash; the host is compared case-insensitively, path and query exactly. A
+   link counts as retrieved when it equals an evidence URL or is a shorter form
+   of one at a `/` or `?` boundary: `https://a.example` is backed by a retrieved
+   `https://a.example/report.pdf`, but `https://a.example/other.pdf` is not.
+4. If at least one link is not backed, the note is appended after a blank line
+   with up to five such links. In a streamed response it arrives as one
+   additional content chunk before the final chunk. If `urls_note` contains no
+   `{urls}`, the links follow the note after a space.
+
+The source line, this note and the number note can be enabled together and are
+appended in that order. `urls_note` without `urls: true` is rejected. The check
+flags, it does not prove: a link that was retrieved can still be cited for a
+claim the page does not make.
+
 ### Requiring a tool call
 
 A system prompt that says "always search first" does not stop a model from