From a93be7cc10b74257eba7d5cbb3b03b5df9d0271c Mon Sep 17 00:00:00 2001 From: Karl-Gustav Kallasmaa <29797627+KGKallasmaa@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:39:25 +0100 Subject: [PATCH] feat(web): add Keenable as a search provider (works without a key) Signed-off-by: Karl-Gustav Kallasmaa <29797627+KGKallasmaa@users.noreply.github.com> --- addons/web/internal/providers/keyed.go | 72 +++++++++++++++++++ addons/web/internal/providers/providers.go | 23 +++--- .../web/internal/providers/providers_test.go | 49 +++++++++++++ addons/web/internal/settings/settings.go | 10 ++- addons/web/internal/settings/settings_test.go | 18 ++++- addons/web/internal/tools/definitions.go | 11 +-- addons/web/internal/tools/tools.go | 6 +- addons/web/manifest.json | 12 +++- addons/web/ui/src/Settings.jsx | 2 +- addons/web/ui/src/index.jsx | 4 +- addons/web/ui/src/lib/model.js | 4 +- addons/web/ui/test/settings.test.mjs | 3 +- 12 files changed, 186 insertions(+), 28 deletions(-) diff --git a/addons/web/internal/providers/keyed.go b/addons/web/internal/providers/keyed.go index 9fa854a845..e7786f7e71 100644 --- a/addons/web/internal/providers/keyed.go +++ b/addons/web/internal/providers/keyed.go @@ -9,6 +9,7 @@ import ( "net/http" "regexp" "strconv" + "strings" "github.com/mudler/LocalAI/pkg/web/search" ) @@ -18,6 +19,9 @@ const ( BraveEndpoint = "https://api.search.brave.com/res/v1/web/search" TavilyEndpoint = "https://api.tavily.com/search" GoogleEndpoint = "https://www.googleapis.com/customsearch/v1" + // KeenableEndpoint is the search with a key; the keyless search is the + // same address with /public appended. + KeenableEndpoint = "https://api.keenable.ai/v1/search" ) // ErrNoKey is returned by a constructor given an empty or malformed key. @@ -211,3 +215,71 @@ func (g *google) Search(ctx context.Context, q search.Query) ([]search.Result, e r := doc.Items return results(len(r), func(i int) (string, string, string) { return r[i].Title, r[i].Link, r[i].Snippet }), nil } + +// --- Keenable ------------------------------------------------------------- + +type keenable struct { + keyed + key string +} + +// KeenableTitle names LocalAI to Keenable's keyless endpoint, which refuses a +// request without an application name. +const KeenableTitle = "LocalAI" + +// NewKeenable returns the Keenable search provider. With a key, the key is +// sent in the X-API-Key header. Without one, the provider calls the keyless +// endpoint (endpoint + "/public"), limited per IP, with the X-Keenable-Title +// header. A key that is set must have the shape of a key. +func NewKeenable(endpoint, key string) (search.Provider, error) { + if key != "" && !ValidKey(key) { + return nil, ErrNoKey + } + if key == "" { + endpoint = strings.TrimSuffix(endpoint, "/") + "/public" + } + k, err := newKeyed(Keenable, "Keenable", endpoint) + if err != nil { + return nil, err + } + return &keenable{keyed: k, key: key}, nil +} + +// Keenable reads a relative delta as "published at most this long ago". +var keenableFreshness = map[string]string{"day": "1d", "week": "7d", "month": "1mo", "year": "1y"} + +func (k *keenable) Search(ctx context.Context, q search.Query) ([]search.Result, error) { + in := map[string]any{"query": q.Text, "max_results": q.MaxResults} + if f := keenableFreshness[q.Freshness]; f != "" { + in["published_after"] = f + } + if q.Site != "" { + in["site"] = q.Site + } + raw, _ := json.Marshal(in) + req, err := newRequest(ctx, http.MethodPost, k.endpoint, raw) + if err != nil { + return nil, err + } + if k.key != "" { + req.Header.Set("X-API-Key", k.key) + } else { + req.Header.Set("X-Keenable-Title", KeenableTitle) + } + body, err := k.do(req) + if err != nil { + return nil, err + } + var doc struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Snippet string `json:"snippet"` + } `json:"results"` + } + if json.Unmarshal(body, &doc) != nil { + return nil, k.badJSON() + } + r := doc.Results + return results(len(r), func(i int) (string, string, string) { return r[i].Title, r[i].URL, r[i].Snippet }), nil +} diff --git a/addons/web/internal/providers/providers.go b/addons/web/internal/providers/providers.go index ea33020034..26fdaa8a03 100644 --- a/addons/web/internal/providers/providers.go +++ b/addons/web/internal/providers/providers.go @@ -1,7 +1,9 @@ // SPDX-License-Identifier: MIT // Package providers holds the search providers of the Web add-on that need a -// key: Brave Search, Tavily and Google Programmable Search. Each implements +// key: Brave Search, Tavily and Google Programmable Search, and Keenable, +// whose key is optional (without one it uses Keenable's keyless endpoint, +// limited per IP). Each implements // search.Provider of pkg/web/search, next to the keyless DuckDuckGo and // SearXNG providers that the built-in web_search uses. // @@ -31,10 +33,12 @@ import ( // The provider names. They are the values of the add-on's order setting. const ( - Brave = "brave" - Tavily = "tavily" - Google = "google" - SearXNG = "searxng" + Brave = "brave" + Tavily = "tavily" + Google = "google" + // Keenable takes a key but does not need one. + Keenable = "keenable" + SearXNG = "searxng" // DuckDuckGo is the keyless entry. Its name stays for the saved // settings; it stands for the whole keyless chain of the built-in // search (DuckDuckGo, then the Brave Search and Bing pages). @@ -43,10 +47,13 @@ const ( // Names lists every provider, in the default order: the keyed ones first, // then SearXNG, then the keyless chain, which needs nothing. -var Names = []string{Brave, Tavily, Google, SearXNG, DuckDuckGo} +var Names = []string{Brave, Tavily, Google, Keenable, SearXNG, DuckDuckGo} -// Keyed reports whether the provider needs a key. -func Keyed(name string) bool { return name == Brave || name == Tavily || name == Google } +// Keyed reports whether the provider takes a key. Keenable takes one but +// also works without it. +func Keyed(name string) bool { + return name == Brave || name == Tavily || name == Google || name == Keenable +} // maxBody bounds the answer of a provider. const maxBody = 1 << 20 diff --git a/addons/web/internal/providers/providers_test.go b/addons/web/internal/providers/providers_test.go index db1cebccba..7865907b4c 100644 --- a/addons/web/internal/providers/providers_test.go +++ b/addons/web/internal/providers/providers_test.go @@ -120,11 +120,59 @@ var _ = Describe("Google Programmable Search", func() { }) }) +var _ = Describe("Keenable", func() { + answer := `{"query":"local ai","mode":"pro","results":[ + {"title":"LocalAI","url":"https://localai.io/","description":"","snippet":"The free, open source alternative."}, + {"title":"bad","url":"ftp://example.org/","description":"","snippet":""}]}` + + It("posts the query with the key in its header, site and freshness, and reads the snippets", func() { + srv, got := fake(200, answer) + p, err := providers.NewKeenable(srv.URL+"/v1/search", key) + Expect(err).NotTo(HaveOccurred()) + res, err := p.Search(context.Background(), q) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal([]search.Result{{Title: "LocalAI", URL: "https://localai.io/", Snippet: "The free, open source alternative."}})) + Expect(got.method).To(Equal(http.MethodPost)) + Expect(got.path).To(Equal("/v1/search")) + Expect(got.header.Get("X-API-Key")).To(Equal(key)) + Expect(got.header.Get("X-Keenable-Title")).To(BeEmpty()) + Expect(got.body).To(HaveKeyWithValue("query", "local ai")) + Expect(got.body).To(HaveKeyWithValue("max_results", BeNumerically("==", 3))) + Expect(got.body).To(HaveKeyWithValue("published_after", "7d")) + Expect(got.body).To(HaveKeyWithValue("site", "github.com")) + Expect(p.Name()).To(Equal("keenable")) + }) + + It("uses the keyless endpoint with the application name when no key is set", func() { + srv, got := fake(200, answer) + p, err := providers.NewKeenable(srv.URL+"/v1/search", "") + Expect(err).NotTo(HaveOccurred()) + res, err := p.Search(context.Background(), search.Query{Text: "local ai", MaxResults: 5}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(HaveLen(1)) + Expect(got.path).To(Equal("/v1/search/public")) + Expect(got.header.Get("X-Keenable-Title")).To(Equal(providers.KeenableTitle)) + Expect(got.header.Get("X-API-Key")).To(BeEmpty()) + Expect(got.body).NotTo(HaveKey("published_after")) + Expect(got.body).NotTo(HaveKey("site")) + }) + + It("refuses a malformed key", func() { + for _, k := range []string{"short", "has space 12345", strings.Repeat("a", 257)} { + _, err := providers.NewKeenable(providers.KeenableEndpoint, k) + Expect(err).To(MatchError(providers.ErrNoKey), k) + } + }) +}) + var _ = Describe("the keyed providers", func() { build := map[string]func(endpoint string) (search.Provider, error){ "brave": func(e string) (search.Provider, error) { return providers.NewBrave(e, key) }, "tavily": func(e string) (search.Provider, error) { return providers.NewTavily(e, key) }, "google": func(e string) (search.Provider, error) { return providers.NewGoogle(e, key, "cx1") }, + "keenable": func(e string) (search.Provider, error) { + return providers.NewKeenable(e, key) + }, } It("refuse a missing or malformed key", func() { @@ -207,6 +255,7 @@ var _ = Describe("the keyed providers", func() { It("list the keyed ones", func() { Expect(providers.Keyed("brave")).To(BeTrue()) + Expect(providers.Keyed("keenable")).To(BeTrue()) Expect(providers.Keyed("duckduckgo")).To(BeFalse()) Expect(providers.ValidEngineID("a b")).To(BeFalse()) }) diff --git a/addons/web/internal/settings/settings.go b/addons/web/internal/settings/settings.go index 53f6807cfb..036b6c1e20 100644 --- a/addons/web/internal/settings/settings.go +++ b/addons/web/internal/settings/settings.go @@ -40,8 +40,12 @@ type Settings struct { } // Defaults are the settings before any choice: every provider in the default -// order, so only the keyless entry is ready. -func Defaults() Settings { return Settings{Order: slices.Clone(providers.Names)} } +// order, so only the keyless entry is ready. Keenable, which is ready without +// a key, stays out of the default order until an administrator adds it, so +// the default search does not change. +func Defaults() Settings { + return Settings{Order: slices.DeleteFunc(slices.Clone(providers.Names), func(n string) bool { return n == providers.Keenable })} +} func (s Settings) clone() Settings { out := Settings{Order: slices.Clone(s.Order), GoogleEngine: s.GoogleEngine, SearXNGURL: s.SearXNGURL} @@ -69,6 +73,8 @@ func (s Settings) Ready(name string) (bool, string) { case s.GoogleEngine == "": return false, "no search engine ID" } + case providers.Keenable: + // Without a key it uses the keyless endpoint. case providers.SearXNG: if s.SearXNGURL == "" { return false, "no address" diff --git a/addons/web/internal/settings/settings_test.go b/addons/web/internal/settings/settings_test.go index 0f56bc7f58..d1d82a5635 100644 --- a/addons/web/internal/settings/settings_test.go +++ b/addons/web/internal/settings/settings_test.go @@ -27,16 +27,28 @@ var _ = Describe("Store", func() { var dir string BeforeEach(func() { dir = GinkgoT().TempDir() }) - It("starts with every provider in order and only DuckDuckGo ready", func() { + It("starts with every provider but Keenable in order and only the keyless ones ready", func() { s, err := settings.Open(dir) Expect(err).NotTo(HaveOccurred()) v := s.View() Expect(v.Order).To(Equal([]string{"brave", "tavily", "google", "searxng", "duckduckgo"})) for _, p := range v.Providers { - Expect(p.Ready).To(Equal(p.Name == "duckduckgo"), p.Name) + Expect(p.Ready).To(Equal(p.Name == "duckduckgo" || p.Name == "keenable"), p.Name) } }) + It("lets Keenable serve without a key, and takes and clears an optional key", func() { + s, _ := settings.Open(dir) + Expect(s.Apply(settings.Change{Order: []string{"keenable", "duckduckgo"}})).To(Succeed()) + Expect(s.View().Providers[3]).To(Equal(settings.ProviderView{Name: "keenable", Keyed: true, Ready: true})) + Expect(s.Apply(settings.Change{Keys: map[string]string{"keenable": key}})).To(Succeed()) + Expect(s.View().Providers[3]).To(Equal(settings.ProviderView{Name: "keenable", Keyed: true, KeySet: true, Ready: true})) + Expect(s.Apply(settings.Change{Clear: []string{"keenable"}})).To(Succeed()) + got, _ := s.Get() + Expect(got.Keys).To(BeEmpty()) + Expect(got.Order).To(Equal([]string{"keenable", "duckduckgo"})) + }) + It("saves a key in a private file and never shows it", func() { s, _ := settings.Open(dir) Expect(s.Apply(settings.Change{Keys: map[string]string{"brave": key}, Order: []string{"brave", "duckduckgo"}})).To(Succeed()) @@ -72,7 +84,7 @@ var _ = Describe("Store", func() { Expect(s.Apply(settings.Change{GoogleEngine: ptr("abc:123"), SearXNGURL: ptr("https://searx.example.org")})).To(Succeed()) v := s.View() Expect(v.Providers[2].Ready).To(BeTrue()) - Expect(v.Providers[3].Ready).To(BeTrue()) + Expect(v.Providers[4].Ready).To(BeTrue()) }) It("counts the revisions", func() { diff --git a/addons/web/internal/tools/definitions.go b/addons/web/internal/tools/definitions.go index bdda74e4d2..3e903819d7 100644 --- a/addons/web/internal/tools/definitions.go +++ b/addons/web/internal/tools/definitions.go @@ -34,7 +34,7 @@ func Definitions() []contract.ToolDefinition { for i, n := range providers.Names { names[i] = n } - keyed := []any{providers.Brave, providers.Tavily, providers.Google} + keyed := []any{providers.Brave, providers.Tavily, providers.Google, providers.Keenable} return []contract.ToolDefinition{ { Name: ToolSearch, Kind: contract.ToolKindRead, Shadows: ShadowsSearch, @@ -75,11 +75,12 @@ func Definitions() []contract.ToolDefinition { "order": obj{"type": "array", "maxItems": len(providers.Names), "items": obj{"type": "string", "enum": names}, "description": "The providers to try, first to last."}, "keys": obj{"type": "object", "additionalProperties": false, "description": "A new key for each named provider. An empty value keeps the saved key.", "properties": obj{ - providers.Brave: obj{"type": "string", "maxLength": 256}, - providers.Tavily: obj{"type": "string", "maxLength": 256}, - providers.Google: obj{"type": "string", "maxLength": 256}, + providers.Brave: obj{"type": "string", "maxLength": 256}, + providers.Tavily: obj{"type": "string", "maxLength": 256}, + providers.Google: obj{"type": "string", "maxLength": 256}, + providers.Keenable: obj{"type": "string", "maxLength": 256}, }}, - "clear": obj{"type": "array", "maxItems": 3, "items": obj{"type": "string", "enum": keyed}, "description": "Remove the saved key of these providers."}, + "clear": obj{"type": "array", "maxItems": len(keyed), "items": obj{"type": "string", "enum": keyed}, "description": "Remove the saved key of these providers."}, "googleEngine": obj{"type": "string", "maxLength": 128, "description": "The Programmable Search Engine ID (cx)."}, "searxngUrl": obj{"type": "string", "maxLength": 2048, "description": "The address of a SearXNG instance."}, }, diff --git a/addons/web/internal/tools/tools.go b/addons/web/internal/tools/tools.go index bc89f57ad6..32ab0d5263 100644 --- a/addons/web/internal/tools/tools.go +++ b/addons/web/internal/tools/tools.go @@ -67,14 +67,14 @@ type Admin interface { // Endpoints are the addresses of the providers. Tests point them at fakes. type Endpoints struct { - Brave, Tavily, Google string + Brave, Tavily, Google, Keenable string // Keyless are the result pages of the keyless chain. Keyless search.KeylessEndpoints } // DefaultEndpoints are the documented addresses. func DefaultEndpoints() Endpoints { - return Endpoints{Brave: providers.BraveEndpoint, Tavily: providers.TavilyEndpoint, Google: providers.GoogleEndpoint, Keyless: search.DefaultKeylessEndpoints()} + return Endpoints{Brave: providers.BraveEndpoint, Tavily: providers.TavilyEndpoint, Google: providers.GoogleEndpoint, Keenable: providers.KeenableEndpoint, Keyless: search.DefaultKeylessEndpoints()} } // Options configure the runtime. @@ -144,6 +144,8 @@ func (r *runtime) build(s settings.Settings) []search.Provider { p, err = providers.NewTavily(r.o.Endpoints.Tavily, s.Keys[n]) case providers.Google: p, err = providers.NewGoogle(r.o.Endpoints.Google, s.Keys[n], s.GoogleEngine) + case providers.Keenable: + p, err = providers.NewKeenable(r.o.Endpoints.Keenable, s.Keys[n]) case providers.SearXNG: p, err = search.NewSearXNG(s.SearXNGURL) case providers.DuckDuckGo: diff --git a/addons/web/manifest.json b/addons/web/manifest.json index 9a8e352bf6..33bdadf858 100644 --- a/addons/web/manifest.json +++ b/addons/web/manifest.json @@ -108,11 +108,12 @@ "enum": [ "brave", "tavily", - "google" + "google", + "keenable" ], "type": "string" }, - "maxItems": 3, + "maxItems": 4, "type": "array" }, "googleEngine": { @@ -132,6 +133,10 @@ "maxLength": 256, "type": "string" }, + "keenable": { + "maxLength": 256, + "type": "string" + }, "tavily": { "maxLength": 256, "type": "string" @@ -146,12 +151,13 @@ "brave", "tavily", "google", + "keenable", "searxng", "duckduckgo" ], "type": "string" }, - "maxItems": 5, + "maxItems": 6, "type": "array" }, "searxngUrl": { diff --git a/addons/web/ui/src/Settings.jsx b/addons/web/ui/src/Settings.jsx index 03bded72bb..5024b28f1c 100644 --- a/addons/web/ui/src/Settings.jsx +++ b/addons/web/ui/src/Settings.jsx @@ -30,7 +30,7 @@ export function createSettings(sdk, t) { )} status={{p.ready ? t('web.settings.ready') : t('web.settings.notReady')}} - meta={!p.ready && p.missing ? p.missing : (p.name === KEYLESS_ENTRY ? t('web.settings.keylessChain') : null)} + meta={!p.ready && p.missing ? p.missing : (p.name === KEYLESS_ENTRY ? t('web.settings.keylessChain') : (p.name === 'keenable' && !p.keySet ? t('web.settings.keenableKeyless') : null))} > {p.on && ( diff --git a/addons/web/ui/src/index.jsx b/addons/web/ui/src/index.jsx index 4b2274faa0..ab73a2309d 100644 --- a/addons/web/ui/src/index.jsx +++ b/addons/web/ui/src/index.jsx @@ -9,19 +9,21 @@ export function createAddon(sdk) { const h = React.createElement sdk.locale.define('en', { 'web.title': 'Web', - 'web.intro': 'Your chats search the web and read web pages through this add-on, in place of the built-in web tools. It can search with Brave Search, Tavily or Google Programmable Search when an administrator sets a key, and it reads the text of PDF files.', + 'web.intro': 'Your chats search the web and read web pages through this add-on, in place of the built-in web tools. It can search with Brave Search, Tavily or Google Programmable Search when an administrator sets a key, or with Keenable, with or without a key, and it reads the text of PDF files.', 'web.rules.order': 'The search providers are tried in the order an administrator chose. When one fails, the next one is tried.', 'web.rules.untrusted': 'What a page says is given to the model as untrusted text. Addresses inside your network are never read.', 'web.rules.settings': 'An administrator sets the providers and their keys on the Settings tab of the Web details, under Add-ons.', 'web.provider.brave': 'Brave Search', 'web.provider.tavily': 'Tavily', 'web.provider.google': 'Google Programmable Search', + 'web.provider.keenable': 'Keenable', 'web.provider.searxng': 'SearXNG', 'web.provider.duckduckgo': 'Search without a key', 'web.keyless.duckduckgo': 'DuckDuckGo page', 'web.keyless.brave_html': 'Brave Search page', 'web.keyless.bing_html': 'Bing page', 'web.settings.keylessChain': 'Tries the result pages of DuckDuckGo, Brave Search and Bing in turn, like the built-in web search. It needs no key.', + 'web.settings.keenableKeyless': 'Works without a key, limited per IP address. A key lifts the limit.', 'web.settings.cooling': '{{name}} is skipped for a few minutes: {{reason}}.', 'web.settings.loading': 'Reading the web settings…', 'web.settings.intro': 'Choose the search providers of web_search and the order they are tried in. A provider without its key or address is skipped. Keys are kept by the add-on and never shown again.', diff --git a/addons/web/ui/src/lib/model.js b/addons/web/ui/src/lib/model.js index 9789e20ffa..ee3a74169d 100644 --- a/addons/web/ui/src/lib/model.js +++ b/addons/web/ui/src/lib/model.js @@ -2,8 +2,8 @@ // status tool answers the settings only to an administrator, and never the // keys: a provider says whether its key is set. -export const PROVIDERS = ['brave', 'tavily', 'google', 'searxng', 'duckduckgo'] -export const KEYED = ['brave', 'tavily', 'google'] +export const PROVIDERS = ['brave', 'tavily', 'google', 'keenable', 'searxng', 'duckduckgo'] +export const KEYED = ['brave', 'tavily', 'google', 'keenable'] // The keyless entry of the order ('duckduckgo', its saved name) is the chain // of the built-in search: these result pages, tried in turn. export const KEYLESS_ENTRY = 'duckduckgo' diff --git a/addons/web/ui/test/settings.test.mjs b/addons/web/ui/test/settings.test.mjs index 4792f3e4b1..127d5eec0f 100644 --- a/addons/web/ui/test/settings.test.mjs +++ b/addons/web/ui/test/settings.test.mjs @@ -14,6 +14,7 @@ const status = (over = {}) => ({ { name: 'brave', keyed: true, keySet: true, ready: true }, { name: 'tavily', keyed: true, keySet: false, ready: false, missing: 'no key' }, { name: 'google', keyed: true, keySet: false, ready: false, missing: 'no key and no search engine ID' }, + { name: 'keenable', keyed: true, keySet: false, ready: true }, { name: 'searxng', keyed: false, keySet: false, ready: false, missing: 'no address' }, { name: 'duckduckgo', keyed: false, keySet: false, ready: true }, ], @@ -34,7 +35,7 @@ test('settings: the providers of the order come first, in order', async () => { const { settingsModel, draftOf, rows, move, toggle } = await lib() const m = settingsModel(status()) let d = draftOf(m) - assert.deepEqual(rows(m, d).map(r => [r.name, r.on]), [['brave', true], ['duckduckgo', true], ['tavily', false], ['google', false], ['searxng', false]]) + assert.deepEqual(rows(m, d).map(r => [r.name, r.on]), [['brave', true], ['duckduckgo', true], ['tavily', false], ['google', false], ['keenable', false], ['searxng', false]]) d = move(d, 'duckduckgo', -1) assert.deepEqual(d.order, ['duckduckgo', 'brave']) assert.deepEqual(move(d, 'duckduckgo', -1).order, ['duckduckgo', 'brave'], 'the first stays first')