feat(web): add Keenable as a search provider (works without a key)

Signed-off-by: Karl-Gustav Kallasmaa <29797627+KGKallasmaa@users.noreply.github.com>

addons/web/internal/providers/keyed.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/mudler/LocalAI/pkg/web/search"
 )
@@ -18,6 +19,9 @@ const (
 	BraveEndpoint  = "https://api.search.brave.com/res/v1/web/search"
 	TavilyEndpoint = "https://api.tavily.com/search"
 	GoogleEndpoint = "https://www.googleapis.com/customsearch/v1"
+	// KeenableEndpoint is the search with a key; the keyless search is the
+	// same address with /public appended.
+	KeenableEndpoint = "https://api.keenable.ai/v1/search"
 )
 
 // ErrNoKey is returned by a constructor given an empty or malformed key.
@@ -211,3 +215,71 @@ func (g *google) Search(ctx context.Context, q search.Query) ([]search.Result, e
 	r := doc.Items
 	return results(len(r), func(i int) (string, string, string) { return r[i].Title, r[i].Link, r[i].Snippet }), nil
 }
+
+// --- Keenable -------------------------------------------------------------
+
+type keenable struct {
+	keyed
+	key string
+}
+
+// KeenableTitle names LocalAI to Keenable's keyless endpoint, which refuses a
+// request without an application name.
+const KeenableTitle = "LocalAI"
+
+// NewKeenable returns the Keenable search provider. With a key, the key is
+// sent in the X-API-Key header. Without one, the provider calls the keyless
+// endpoint (endpoint + "/public"), limited per IP, with the X-Keenable-Title
+// header. A key that is set must have the shape of a key.
+func NewKeenable(endpoint, key string) (search.Provider, error) {
+	if key != "" && !ValidKey(key) {
+		return nil, ErrNoKey
+	}
+	if key == "" {
+		endpoint = strings.TrimSuffix(endpoint, "/") + "/public"
+	}
+	k, err := newKeyed(Keenable, "Keenable", endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &keenable{keyed: k, key: key}, nil
+}
+
+// Keenable reads a relative delta as "published at most this long ago".
+var keenableFreshness = map[string]string{"day": "1d", "week": "7d", "month": "1mo", "year": "1y"}
+
+func (k *keenable) Search(ctx context.Context, q search.Query) ([]search.Result, error) {
+	in := map[string]any{"query": q.Text, "max_results": q.MaxResults}
+	if f := keenableFreshness[q.Freshness]; f != "" {
+		in["published_after"] = f
+	}
+	if q.Site != "" {
+		in["site"] = q.Site
+	}
+	raw, _ := json.Marshal(in)
+	req, err := newRequest(ctx, http.MethodPost, k.endpoint, raw)
+	if err != nil {
+		return nil, err
+	}
+	if k.key != "" {
+		req.Header.Set("X-API-Key", k.key)
+	} else {
+		req.Header.Set("X-Keenable-Title", KeenableTitle)
+	}
+	body, err := k.do(req)
+	if err != nil {
+		return nil, err
+	}
+	var doc struct {
+		Results []struct {
+			Title   string `json:"title"`
+			URL     string `json:"url"`
+			Snippet string `json:"snippet"`
+		} `json:"results"`
+	}
+	if json.Unmarshal(body, &doc) != nil {
+		return nil, k.badJSON()
+	}
+	r := doc.Results
+	return results(len(r), func(i int) (string, string, string) { return r[i].Title, r[i].URL, r[i].Snippet }), nil
+}

addons/web/internal/providers/providers.go

@@ -1,7 +1,9 @@
 // SPDX-License-Identifier: MIT
 
 // Package providers holds the search providers of the Web add-on that need a
-// key: Brave Search, Tavily and Google Programmable Search. Each implements
+// key: Brave Search, Tavily and Google Programmable Search, and Keenable,
+// whose key is optional (without one it uses Keenable's keyless endpoint,
+// limited per IP). Each implements
 // search.Provider of pkg/web/search, next to the keyless DuckDuckGo and
 // SearXNG providers that the built-in web_search uses.
 //
@@ -31,10 +33,12 @@ import (
 
 // The provider names. They are the values of the add-on's order setting.
 const (
-	Brave   = "brave"
-	Tavily  = "tavily"
-	Google  = "google"
-	SearXNG = "searxng"
+	Brave  = "brave"
+	Tavily = "tavily"
+	Google = "google"
+	// Keenable takes a key but does not need one.
+	Keenable = "keenable"
+	SearXNG  = "searxng"
 	// DuckDuckGo is the keyless entry. Its name stays for the saved
 	// settings; it stands for the whole keyless chain of the built-in
 	// search (DuckDuckGo, then the Brave Search and Bing pages).
@@ -43,10 +47,13 @@ const (
 
 // Names lists every provider, in the default order: the keyed ones first,
 // then SearXNG, then the keyless chain, which needs nothing.
-var Names = []string{Brave, Tavily, Google, SearXNG, DuckDuckGo}
+var Names = []string{Brave, Tavily, Google, Keenable, SearXNG, DuckDuckGo}
 
-// Keyed reports whether the provider needs a key.
-func Keyed(name string) bool { return name == Brave || name == Tavily || name == Google }
+// Keyed reports whether the provider takes a key. Keenable takes one but
+// also works without it.
+func Keyed(name string) bool {
+	return name == Brave || name == Tavily || name == Google || name == Keenable
+}
 
 // maxBody bounds the answer of a provider.
 const maxBody = 1 << 20

addons/web/internal/providers/providers_test.go

@@ -120,11 +120,59 @@ var _ = Describe("Google Programmable Search", func() {
 	})
 })
 
+var _ = Describe("Keenable", func() {
+	answer := `{"query":"local ai","mode":"pro","results":[
+		{"title":"LocalAI","url":"https://localai.io/","description":"","snippet":"The free, open source alternative."},
+		{"title":"bad","url":"ftp://example.org/","description":"","snippet":""}]}`
+
+	It("posts the query with the key in its header, site and freshness, and reads the snippets", func() {
+		srv, got := fake(200, answer)
+		p, err := providers.NewKeenable(srv.URL+"/v1/search", key)
+		Expect(err).NotTo(HaveOccurred())
+		res, err := p.Search(context.Background(), q)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res).To(Equal([]search.Result{{Title: "LocalAI", URL: "https://localai.io/", Snippet: "The free, open source alternative."}}))
+		Expect(got.method).To(Equal(http.MethodPost))
+		Expect(got.path).To(Equal("/v1/search"))
+		Expect(got.header.Get("X-API-Key")).To(Equal(key))
+		Expect(got.header.Get("X-Keenable-Title")).To(BeEmpty())
+		Expect(got.body).To(HaveKeyWithValue("query", "local ai"))
+		Expect(got.body).To(HaveKeyWithValue("max_results", BeNumerically("==", 3)))
+		Expect(got.body).To(HaveKeyWithValue("published_after", "7d"))
+		Expect(got.body).To(HaveKeyWithValue("site", "github.com"))
+		Expect(p.Name()).To(Equal("keenable"))
+	})
+
+	It("uses the keyless endpoint with the application name when no key is set", func() {
+		srv, got := fake(200, answer)
+		p, err := providers.NewKeenable(srv.URL+"/v1/search", "")
+		Expect(err).NotTo(HaveOccurred())
+		res, err := p.Search(context.Background(), search.Query{Text: "local ai", MaxResults: 5})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res).To(HaveLen(1))
+		Expect(got.path).To(Equal("/v1/search/public"))
+		Expect(got.header.Get("X-Keenable-Title")).To(Equal(providers.KeenableTitle))
+		Expect(got.header.Get("X-API-Key")).To(BeEmpty())
+		Expect(got.body).NotTo(HaveKey("published_after"))
+		Expect(got.body).NotTo(HaveKey("site"))
+	})
+
+	It("refuses a malformed key", func() {
+		for _, k := range []string{"short", "has space 12345", strings.Repeat("a", 257)} {
+			_, err := providers.NewKeenable(providers.KeenableEndpoint, k)
+			Expect(err).To(MatchError(providers.ErrNoKey), k)
+		}
+	})
+})
+
 var _ = Describe("the keyed providers", func() {
 	build := map[string]func(endpoint string) (search.Provider, error){
 		"brave":  func(e string) (search.Provider, error) { return providers.NewBrave(e, key) },
 		"tavily": func(e string) (search.Provider, error) { return providers.NewTavily(e, key) },
 		"google": func(e string) (search.Provider, error) { return providers.NewGoogle(e, key, "cx1") },
+		"keenable": func(e string) (search.Provider, error) {
+			return providers.NewKeenable(e, key)
+		},
 	}
 
 	It("refuse a missing or malformed key", func() {
@@ -207,6 +255,7 @@ var _ = Describe("the keyed providers", func() {
 
 	It("list the keyed ones", func() {
 		Expect(providers.Keyed("brave")).To(BeTrue())
+		Expect(providers.Keyed("keenable")).To(BeTrue())
 		Expect(providers.Keyed("duckduckgo")).To(BeFalse())
 		Expect(providers.ValidEngineID("a b")).To(BeFalse())
 	})

addons/web/internal/settings/settings.go

@@ -40,8 +40,12 @@ type Settings struct {
 }
 
 // Defaults are the settings before any choice: every provider in the default
-// order, so only the keyless entry is ready.
-func Defaults() Settings { return Settings{Order: slices.Clone(providers.Names)} }
+// order, so only the keyless entry is ready. Keenable, which is ready without
+// a key, stays out of the default order until an administrator adds it, so
+// the default search does not change.
+func Defaults() Settings {
+	return Settings{Order: slices.DeleteFunc(slices.Clone(providers.Names), func(n string) bool { return n == providers.Keenable })}
+}
 
 func (s Settings) clone() Settings {
 	out := Settings{Order: slices.Clone(s.Order), GoogleEngine: s.GoogleEngine, SearXNGURL: s.SearXNGURL}
@@ -69,6 +73,8 @@ func (s Settings) Ready(name string) (bool, string) {
 		case s.GoogleEngine == "":
 			return false, "no search engine ID"
 		}
+	case providers.Keenable:
+		// Without a key it uses the keyless endpoint.
 	case providers.SearXNG:
 		if s.SearXNGURL == "" {
 			return false, "no address"

addons/web/internal/settings/settings_test.go

@@ -27,16 +27,28 @@ var _ = Describe("Store", func() {
 	var dir string
 	BeforeEach(func() { dir = GinkgoT().TempDir() })
 
-	It("starts with every provider in order and only DuckDuckGo ready", func() {
+	It("starts with every provider but Keenable in order and only the keyless ones ready", func() {
 		s, err := settings.Open(dir)
 		Expect(err).NotTo(HaveOccurred())
 		v := s.View()
 		Expect(v.Order).To(Equal([]string{"brave", "tavily", "google", "searxng", "duckduckgo"}))
 		for _, p := range v.Providers {
-			Expect(p.Ready).To(Equal(p.Name == "duckduckgo"), p.Name)
+			Expect(p.Ready).To(Equal(p.Name == "duckduckgo" || p.Name == "keenable"), p.Name)
 		}
 	})
 
+	It("lets Keenable serve without a key, and takes and clears an optional key", func() {
+		s, _ := settings.Open(dir)
+		Expect(s.Apply(settings.Change{Order: []string{"keenable", "duckduckgo"}})).To(Succeed())
+		Expect(s.View().Providers[3]).To(Equal(settings.ProviderView{Name: "keenable", Keyed: true, Ready: true}))
+		Expect(s.Apply(settings.Change{Keys: map[string]string{"keenable": key}})).To(Succeed())
+		Expect(s.View().Providers[3]).To(Equal(settings.ProviderView{Name: "keenable", Keyed: true, KeySet: true, Ready: true}))
+		Expect(s.Apply(settings.Change{Clear: []string{"keenable"}})).To(Succeed())
+		got, _ := s.Get()
+		Expect(got.Keys).To(BeEmpty())
+		Expect(got.Order).To(Equal([]string{"keenable", "duckduckgo"}))
+	})
+
 	It("saves a key in a private file and never shows it", func() {
 		s, _ := settings.Open(dir)
 		Expect(s.Apply(settings.Change{Keys: map[string]string{"brave": key}, Order: []string{"brave", "duckduckgo"}})).To(Succeed())
@@ -72,7 +84,7 @@ var _ = Describe("Store", func() {
 		Expect(s.Apply(settings.Change{GoogleEngine: ptr("abc:123"), SearXNGURL: ptr("https://searx.example.org")})).To(Succeed())
 		v := s.View()
 		Expect(v.Providers[2].Ready).To(BeTrue())
-		Expect(v.Providers[3].Ready).To(BeTrue())
+		Expect(v.Providers[4].Ready).To(BeTrue())
 	})
 
 	It("counts the revisions", func() {

addons/web/internal/tools/definitions.go

@@ -34,7 +34,7 @@ func Definitions() []contract.ToolDefinition {
 	for i, n := range providers.Names {
 		names[i] = n
 	}
-	keyed := []any{providers.Brave, providers.Tavily, providers.Google}
+	keyed := []any{providers.Brave, providers.Tavily, providers.Google, providers.Keenable}
 	return []contract.ToolDefinition{
 		{
 			Name: ToolSearch, Kind: contract.ToolKindRead, Shadows: ShadowsSearch,
@@ -75,11 +75,12 @@ func Definitions() []contract.ToolDefinition {
 					"order": obj{"type": "array", "maxItems": len(providers.Names), "items": obj{"type": "string", "enum": names}, "description": "The providers to try, first to last."},
 					"keys": obj{"type": "object", "additionalProperties": false, "description": "A new key for each named provider. An empty value keeps the saved key.",
 						"properties": obj{
-							providers.Brave:  obj{"type": "string", "maxLength": 256},
-							providers.Tavily: obj{"type": "string", "maxLength": 256},
-							providers.Google: obj{"type": "string", "maxLength": 256},
+							providers.Brave:    obj{"type": "string", "maxLength": 256},
+							providers.Tavily:   obj{"type": "string", "maxLength": 256},
+							providers.Google:   obj{"type": "string", "maxLength": 256},
+							providers.Keenable: obj{"type": "string", "maxLength": 256},
 						}},
-					"clear":        obj{"type": "array", "maxItems": 3, "items": obj{"type": "string", "enum": keyed}, "description": "Remove the saved key of these providers."},
+					"clear":        obj{"type": "array", "maxItems": len(keyed), "items": obj{"type": "string", "enum": keyed}, "description": "Remove the saved key of these providers."},
 					"googleEngine": obj{"type": "string", "maxLength": 128, "description": "The Programmable Search Engine ID (cx)."},
 					"searxngUrl":   obj{"type": "string", "maxLength": 2048, "description": "The address of a SearXNG instance."},
 				},

addons/web/internal/tools/tools.go

@@ -67,14 +67,14 @@ type Admin interface {
 
 // Endpoints are the addresses of the providers. Tests point them at fakes.
 type Endpoints struct {
-	Brave, Tavily, Google string
+	Brave, Tavily, Google, Keenable string
 	// Keyless are the result pages of the keyless chain.
 	Keyless search.KeylessEndpoints
 }
 
 // DefaultEndpoints are the documented addresses.
 func DefaultEndpoints() Endpoints {
-	return Endpoints{Brave: providers.BraveEndpoint, Tavily: providers.TavilyEndpoint, Google: providers.GoogleEndpoint, Keyless: search.DefaultKeylessEndpoints()}
+	return Endpoints{Brave: providers.BraveEndpoint, Tavily: providers.TavilyEndpoint, Google: providers.GoogleEndpoint, Keenable: providers.KeenableEndpoint, Keyless: search.DefaultKeylessEndpoints()}
 }
 
 // Options configure the runtime.
@@ -144,6 +144,8 @@ func (r *runtime) build(s settings.Settings) []search.Provider {
 			p, err = providers.NewTavily(r.o.Endpoints.Tavily, s.Keys[n])
 		case providers.Google:
 			p, err = providers.NewGoogle(r.o.Endpoints.Google, s.Keys[n], s.GoogleEngine)
+		case providers.Keenable:
+			p, err = providers.NewKeenable(r.o.Endpoints.Keenable, s.Keys[n])
 		case providers.SearXNG:
 			p, err = search.NewSearXNG(s.SearXNGURL)
 		case providers.DuckDuckGo:

addons/web/manifest.json

@@ -108,11 +108,12 @@
                 "enum": [
                   "brave",
                   "tavily",
-                  "google"
+                  "google",
+                  "keenable"
                 ],
                 "type": "string"
               },
-              "maxItems": 3,
+              "maxItems": 4,
               "type": "array"
             },
             "googleEngine": {
@@ -132,6 +133,10 @@
                   "maxLength": 256,
                   "type": "string"
                 },
+                "keenable": {
+                  "maxLength": 256,
+                  "type": "string"
+                },
                 "tavily": {
                   "maxLength": 256,
                   "type": "string"
@@ -146,12 +151,13 @@
                   "brave",
                   "tavily",
                   "google",
+                  "keenable",
                   "searxng",
                   "duckduckgo"
                 ],
                 "type": "string"
               },
-              "maxItems": 5,
+              "maxItems": 6,
               "type": "array"
             },
             "searxngUrl": {

addons/web/ui/src/Settings.jsx

@@ -30,7 +30,7 @@ export function createSettings(sdk, t) {
           </span>
         )}
         status={<StatusChip tone={p.ready ? 'ok' : 'idle'}>{p.ready ? t('web.settings.ready') : t('web.settings.notReady')}</StatusChip>}
-        meta={!p.ready && p.missing ? p.missing : (p.name === KEYLESS_ENTRY ? t('web.settings.keylessChain') : null)}
+        meta={!p.ready && p.missing ? p.missing : (p.name === KEYLESS_ENTRY ? t('web.settings.keylessChain') : (p.name === 'keenable' && !p.keySet ? t('web.settings.keenableKeyless') : null))}
       >
         {p.on && (
           <Actions>

addons/web/ui/src/index.jsx

@@ -9,19 +9,21 @@ export function createAddon(sdk) {
   const h = React.createElement
   sdk.locale.define('en', {
     'web.title': 'Web',
-    'web.intro': 'Your chats search the web and read web pages through this add-on, in place of the built-in web tools. It can search with Brave Search, Tavily or Google Programmable Search when an administrator sets a key, and it reads the text of PDF files.',
+    'web.intro': 'Your chats search the web and read web pages through this add-on, in place of the built-in web tools. It can search with Brave Search, Tavily or Google Programmable Search when an administrator sets a key, or with Keenable, with or without a key, and it reads the text of PDF files.',
     'web.rules.order': 'The search providers are tried in the order an administrator chose. When one fails, the next one is tried.',
     'web.rules.untrusted': 'What a page says is given to the model as untrusted text. Addresses inside your network are never read.',
     'web.rules.settings': 'An administrator sets the providers and their keys on the Settings tab of the Web details, under Add-ons.',
     'web.provider.brave': 'Brave Search',
     'web.provider.tavily': 'Tavily',
     'web.provider.google': 'Google Programmable Search',
+    'web.provider.keenable': 'Keenable',
     'web.provider.searxng': 'SearXNG',
     'web.provider.duckduckgo': 'Search without a key',
     'web.keyless.duckduckgo': 'DuckDuckGo page',
     'web.keyless.brave_html': 'Brave Search page',
     'web.keyless.bing_html': 'Bing page',
     'web.settings.keylessChain': 'Tries the result pages of DuckDuckGo, Brave Search and Bing in turn, like the built-in web search. It needs no key.',
+    'web.settings.keenableKeyless': 'Works without a key, limited per IP address. A key lifts the limit.',
     'web.settings.cooling': '{{name}} is skipped for a few minutes: {{reason}}.',
     'web.settings.loading': 'Reading the web settings…',
     'web.settings.intro': 'Choose the search providers of web_search and the order they are tried in. A provider without its key or address is skipped. Keys are kept by the add-on and never shown again.',

addons/web/ui/src/lib/model.js

@@ -2,8 +2,8 @@
 // status tool answers the settings only to an administrator, and never the
 // keys: a provider says whether its key is set.
 
-export const PROVIDERS = ['brave', 'tavily', 'google', 'searxng', 'duckduckgo']
-export const KEYED = ['brave', 'tavily', 'google']
+export const PROVIDERS = ['brave', 'tavily', 'google', 'keenable', 'searxng', 'duckduckgo']
+export const KEYED = ['brave', 'tavily', 'google', 'keenable']
 // The keyless entry of the order ('duckduckgo', its saved name) is the chain
 // of the built-in search: these result pages, tried in turn.
 export const KEYLESS_ENTRY = 'duckduckgo'

addons/web/ui/test/settings.test.mjs

@@ -14,6 +14,7 @@ const status = (over = {}) => ({
       { name: 'brave', keyed: true, keySet: true, ready: true },
       { name: 'tavily', keyed: true, keySet: false, ready: false, missing: 'no key' },
       { name: 'google', keyed: true, keySet: false, ready: false, missing: 'no key and no search engine ID' },
+      { name: 'keenable', keyed: true, keySet: false, ready: true },
       { name: 'searxng', keyed: false, keySet: false, ready: false, missing: 'no address' },
       { name: 'duckduckgo', keyed: false, keySet: false, ready: true },
     ],
@@ -34,7 +35,7 @@ test('settings: the providers of the order come first, in order', async () => {
   const { settingsModel, draftOf, rows, move, toggle } = await lib()
   const m = settingsModel(status())
   let d = draftOf(m)
-  assert.deepEqual(rows(m, d).map(r => [r.name, r.on]), [['brave', true], ['duckduckgo', true], ['tavily', false], ['google', false], ['searxng', false]])
+  assert.deepEqual(rows(m, d).map(r => [r.name, r.on]), [['brave', true], ['duckduckgo', true], ['tavily', false], ['google', false], ['keenable', false], ['searxng', false]])
   d = move(d, 'duckduckgo', -1)
   assert.deepEqual(d.order, ['duckduckgo', 'brave'])
   assert.deepEqual(move(d, 'duckduckgo', -1).order, ['duckduckgo', 'brave'], 'the first stays first')