diff --git a/addons/web/internal/providers/keyed.go b/addons/web/internal/providers/keyed.go
index 9fa854a845..e7786f7e71 100644
--- a/addons/web/internal/providers/keyed.go
+++ b/addons/web/internal/providers/keyed.go
@@ -9,6 +9,7 @@ import (
"net/http"
"regexp"
"strconv"
+ "strings"
"github.com/mudler/LocalAI/pkg/web/search"
)
@@ -18,6 +19,9 @@ const (
BraveEndpoint = "https://api.search.brave.com/res/v1/web/search"
TavilyEndpoint = "https://api.tavily.com/search"
GoogleEndpoint = "https://www.googleapis.com/customsearch/v1"
+ // KeenableEndpoint is the search with a key; the keyless search is the
+ // same address with /public appended.
+ KeenableEndpoint = "https://api.keenable.ai/v1/search"
)
// ErrNoKey is returned by a constructor given an empty or malformed key.
@@ -211,3 +215,71 @@ func (g *google) Search(ctx context.Context, q search.Query) ([]search.Result, e
r := doc.Items
return results(len(r), func(i int) (string, string, string) { return r[i].Title, r[i].Link, r[i].Snippet }), nil
}
+
+// --- Keenable -------------------------------------------------------------
+
+type keenable struct {
+ keyed
+ key string
+}
+
+// KeenableTitle names LocalAI to Keenable's keyless endpoint, which refuses a
+// request without an application name.
+const KeenableTitle = "LocalAI"
+
+// NewKeenable returns the Keenable search provider. With a key, the key is
+// sent in the X-API-Key header. Without one, the provider calls the keyless
+// endpoint (endpoint + "/public"), limited per IP, with the X-Keenable-Title
+// header. A key that is set must have the shape of a key.
+func NewKeenable(endpoint, key string) (search.Provider, error) {
+ if key != "" && !ValidKey(key) {
+ return nil, ErrNoKey
+ }
+ if key == "" {
+ endpoint = strings.TrimSuffix(endpoint, "/") + "/public"
+ }
+ k, err := newKeyed(Keenable, "Keenable", endpoint)
+ if err != nil {
+ return nil, err
+ }
+ return &keenable{keyed: k, key: key}, nil
+}
+
+// Keenable reads a relative delta as "published at most this long ago".
+var keenableFreshness = map[string]string{"day": "1d", "week": "7d", "month": "1mo", "year": "1y"}
+
+func (k *keenable) Search(ctx context.Context, q search.Query) ([]search.Result, error) {
+ in := map[string]any{"query": q.Text, "max_results": q.MaxResults}
+ if f := keenableFreshness[q.Freshness]; f != "" {
+ in["published_after"] = f
+ }
+ if q.Site != "" {
+ in["site"] = q.Site
+ }
+ raw, _ := json.Marshal(in)
+ req, err := newRequest(ctx, http.MethodPost, k.endpoint, raw)
+ if err != nil {
+ return nil, err
+ }
+ if k.key != "" {
+ req.Header.Set("X-API-Key", k.key)
+ } else {
+ req.Header.Set("X-Keenable-Title", KeenableTitle)
+ }
+ body, err := k.do(req)
+ if err != nil {
+ return nil, err
+ }
+ var doc struct {
+ Results []struct {
+ Title string `json:"title"`
+ URL string `json:"url"`
+ Snippet string `json:"snippet"`
+ } `json:"results"`
+ }
+ if json.Unmarshal(body, &doc) != nil {
+ return nil, k.badJSON()
+ }
+ r := doc.Results
+ return results(len(r), func(i int) (string, string, string) { return r[i].Title, r[i].URL, r[i].Snippet }), nil
+}
diff --git a/addons/web/internal/providers/providers.go b/addons/web/internal/providers/providers.go
index ea33020034..26fdaa8a03 100644
--- a/addons/web/internal/providers/providers.go
+++ b/addons/web/internal/providers/providers.go
@@ -1,7 +1,9 @@
// SPDX-License-Identifier: MIT
// Package providers holds the search providers of the Web add-on that need a
-// key: Brave Search, Tavily and Google Programmable Search. Each implements
+// key: Brave Search, Tavily and Google Programmable Search, and Keenable,
+// whose key is optional (without one it uses Keenable's keyless endpoint,
+// limited per IP). Each implements
// search.Provider of pkg/web/search, next to the keyless DuckDuckGo and
// SearXNG providers that the built-in web_search uses.
//
@@ -31,10 +33,12 @@ import (
// The provider names. They are the values of the add-on's order setting.
const (
- Brave = "brave"
- Tavily = "tavily"
- Google = "google"
- SearXNG = "searxng"
+ Brave = "brave"
+ Tavily = "tavily"
+ Google = "google"
+ // Keenable takes a key but does not need one.
+ Keenable = "keenable"
+ SearXNG = "searxng"
// DuckDuckGo is the keyless entry. Its name stays for the saved
// settings; it stands for the whole keyless chain of the built-in
// search (DuckDuckGo, then the Brave Search and Bing pages).
@@ -43,10 +47,13 @@ const (
// Names lists every provider, in the default order: the keyed ones first,
// then SearXNG, then the keyless chain, which needs nothing.
-var Names = []string{Brave, Tavily, Google, SearXNG, DuckDuckGo}
+var Names = []string{Brave, Tavily, Google, Keenable, SearXNG, DuckDuckGo}
-// Keyed reports whether the provider needs a key.
-func Keyed(name string) bool { return name == Brave || name == Tavily || name == Google }
+// Keyed reports whether the provider takes a key. Keenable takes one but
+// also works without it.
+func Keyed(name string) bool {
+ return name == Brave || name == Tavily || name == Google || name == Keenable
+}
// maxBody bounds the answer of a provider.
const maxBody = 1 << 20
diff --git a/addons/web/internal/providers/providers_test.go b/addons/web/internal/providers/providers_test.go
index db1cebccba..7865907b4c 100644
--- a/addons/web/internal/providers/providers_test.go
+++ b/addons/web/internal/providers/providers_test.go
@@ -120,11 +120,59 @@ var _ = Describe("Google Programmable Search", func() {
})
})
+var _ = Describe("Keenable", func() {
+ answer := `{"query":"local ai","mode":"pro","results":[
+ {"title":"LocalAI","url":"https://localai.io/","description":"","snippet":"The free, open source alternative."},
+ {"title":"bad","url":"ftp://example.org/","description":"","snippet":""}]}`
+
+ It("posts the query with the key in its header, site and freshness, and reads the snippets", func() {
+ srv, got := fake(200, answer)
+ p, err := providers.NewKeenable(srv.URL+"/v1/search", key)
+ Expect(err).NotTo(HaveOccurred())
+ res, err := p.Search(context.Background(), q)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(res).To(Equal([]search.Result{{Title: "LocalAI", URL: "https://localai.io/", Snippet: "The free, open source alternative."}}))
+ Expect(got.method).To(Equal(http.MethodPost))
+ Expect(got.path).To(Equal("/v1/search"))
+ Expect(got.header.Get("X-API-Key")).To(Equal(key))
+ Expect(got.header.Get("X-Keenable-Title")).To(BeEmpty())
+ Expect(got.body).To(HaveKeyWithValue("query", "local ai"))
+ Expect(got.body).To(HaveKeyWithValue("max_results", BeNumerically("==", 3)))
+ Expect(got.body).To(HaveKeyWithValue("published_after", "7d"))
+ Expect(got.body).To(HaveKeyWithValue("site", "github.com"))
+ Expect(p.Name()).To(Equal("keenable"))
+ })
+
+ It("uses the keyless endpoint with the application name when no key is set", func() {
+ srv, got := fake(200, answer)
+ p, err := providers.NewKeenable(srv.URL+"/v1/search", "")
+ Expect(err).NotTo(HaveOccurred())
+ res, err := p.Search(context.Background(), search.Query{Text: "local ai", MaxResults: 5})
+ Expect(err).NotTo(HaveOccurred())
+ Expect(res).To(HaveLen(1))
+ Expect(got.path).To(Equal("/v1/search/public"))
+ Expect(got.header.Get("X-Keenable-Title")).To(Equal(providers.KeenableTitle))
+ Expect(got.header.Get("X-API-Key")).To(BeEmpty())
+ Expect(got.body).NotTo(HaveKey("published_after"))
+ Expect(got.body).NotTo(HaveKey("site"))
+ })
+
+ It("refuses a malformed key", func() {
+ for _, k := range []string{"short", "has space 12345", strings.Repeat("a", 257)} {
+ _, err := providers.NewKeenable(providers.KeenableEndpoint, k)
+ Expect(err).To(MatchError(providers.ErrNoKey), k)
+ }
+ })
+})
+
var _ = Describe("the keyed providers", func() {
build := map[string]func(endpoint string) (search.Provider, error){
"brave": func(e string) (search.Provider, error) { return providers.NewBrave(e, key) },
"tavily": func(e string) (search.Provider, error) { return providers.NewTavily(e, key) },
"google": func(e string) (search.Provider, error) { return providers.NewGoogle(e, key, "cx1") },
+ "keenable": func(e string) (search.Provider, error) {
+ return providers.NewKeenable(e, key)
+ },
}
It("refuse a missing or malformed key", func() {
@@ -207,6 +255,7 @@ var _ = Describe("the keyed providers", func() {
It("list the keyed ones", func() {
Expect(providers.Keyed("brave")).To(BeTrue())
+ Expect(providers.Keyed("keenable")).To(BeTrue())
Expect(providers.Keyed("duckduckgo")).To(BeFalse())
Expect(providers.ValidEngineID("a b")).To(BeFalse())
})
diff --git a/addons/web/internal/settings/settings.go b/addons/web/internal/settings/settings.go
index 53f6807cfb..036b6c1e20 100644
--- a/addons/web/internal/settings/settings.go
+++ b/addons/web/internal/settings/settings.go
@@ -40,8 +40,12 @@ type Settings struct {
}
// Defaults are the settings before any choice: every provider in the default
-// order, so only the keyless entry is ready.
-func Defaults() Settings { return Settings{Order: slices.Clone(providers.Names)} }
+// order, so only the keyless entry is ready. Keenable, which is ready without
+// a key, stays out of the default order until an administrator adds it, so
+// the default search does not change.
+func Defaults() Settings {
+ return Settings{Order: slices.DeleteFunc(slices.Clone(providers.Names), func(n string) bool { return n == providers.Keenable })}
+}
func (s Settings) clone() Settings {
out := Settings{Order: slices.Clone(s.Order), GoogleEngine: s.GoogleEngine, SearXNGURL: s.SearXNGURL}
@@ -69,6 +73,8 @@ func (s Settings) Ready(name string) (bool, string) {
case s.GoogleEngine == "":
return false, "no search engine ID"
}
+ case providers.Keenable:
+ // Without a key it uses the keyless endpoint.
case providers.SearXNG:
if s.SearXNGURL == "" {
return false, "no address"
diff --git a/addons/web/internal/settings/settings_test.go b/addons/web/internal/settings/settings_test.go
index 0f56bc7f58..d1d82a5635 100644
--- a/addons/web/internal/settings/settings_test.go
+++ b/addons/web/internal/settings/settings_test.go
@@ -27,16 +27,28 @@ var _ = Describe("Store", func() {
var dir string
BeforeEach(func() { dir = GinkgoT().TempDir() })
- It("starts with every provider in order and only DuckDuckGo ready", func() {
+ It("starts with every provider but Keenable in order and only the keyless ones ready", func() {
s, err := settings.Open(dir)
Expect(err).NotTo(HaveOccurred())
v := s.View()
Expect(v.Order).To(Equal([]string{"brave", "tavily", "google", "searxng", "duckduckgo"}))
for _, p := range v.Providers {
- Expect(p.Ready).To(Equal(p.Name == "duckduckgo"), p.Name)
+ Expect(p.Ready).To(Equal(p.Name == "duckduckgo" || p.Name == "keenable"), p.Name)
}
})
+ It("lets Keenable serve without a key, and takes and clears an optional key", func() {
+ s, _ := settings.Open(dir)
+ Expect(s.Apply(settings.Change{Order: []string{"keenable", "duckduckgo"}})).To(Succeed())
+ Expect(s.View().Providers[3]).To(Equal(settings.ProviderView{Name: "keenable", Keyed: true, Ready: true}))
+ Expect(s.Apply(settings.Change{Keys: map[string]string{"keenable": key}})).To(Succeed())
+ Expect(s.View().Providers[3]).To(Equal(settings.ProviderView{Name: "keenable", Keyed: true, KeySet: true, Ready: true}))
+ Expect(s.Apply(settings.Change{Clear: []string{"keenable"}})).To(Succeed())
+ got, _ := s.Get()
+ Expect(got.Keys).To(BeEmpty())
+ Expect(got.Order).To(Equal([]string{"keenable", "duckduckgo"}))
+ })
+
It("saves a key in a private file and never shows it", func() {
s, _ := settings.Open(dir)
Expect(s.Apply(settings.Change{Keys: map[string]string{"brave": key}, Order: []string{"brave", "duckduckgo"}})).To(Succeed())
@@ -72,7 +84,7 @@ var _ = Describe("Store", func() {
Expect(s.Apply(settings.Change{GoogleEngine: ptr("abc:123"), SearXNGURL: ptr("https://searx.example.org")})).To(Succeed())
v := s.View()
Expect(v.Providers[2].Ready).To(BeTrue())
- Expect(v.Providers[3].Ready).To(BeTrue())
+ Expect(v.Providers[4].Ready).To(BeTrue())
})
It("counts the revisions", func() {
diff --git a/addons/web/internal/tools/definitions.go b/addons/web/internal/tools/definitions.go
index bdda74e4d2..3e903819d7 100644
--- a/addons/web/internal/tools/definitions.go
+++ b/addons/web/internal/tools/definitions.go
@@ -34,7 +34,7 @@ func Definitions() []contract.ToolDefinition {
for i, n := range providers.Names {
names[i] = n
}
- keyed := []any{providers.Brave, providers.Tavily, providers.Google}
+ keyed := []any{providers.Brave, providers.Tavily, providers.Google, providers.Keenable}
return []contract.ToolDefinition{
{
Name: ToolSearch, Kind: contract.ToolKindRead, Shadows: ShadowsSearch,
@@ -75,11 +75,12 @@ func Definitions() []contract.ToolDefinition {
"order": obj{"type": "array", "maxItems": len(providers.Names), "items": obj{"type": "string", "enum": names}, "description": "The providers to try, first to last."},
"keys": obj{"type": "object", "additionalProperties": false, "description": "A new key for each named provider. An empty value keeps the saved key.",
"properties": obj{
- providers.Brave: obj{"type": "string", "maxLength": 256},
- providers.Tavily: obj{"type": "string", "maxLength": 256},
- providers.Google: obj{"type": "string", "maxLength": 256},
+ providers.Brave: obj{"type": "string", "maxLength": 256},
+ providers.Tavily: obj{"type": "string", "maxLength": 256},
+ providers.Google: obj{"type": "string", "maxLength": 256},
+ providers.Keenable: obj{"type": "string", "maxLength": 256},
}},
- "clear": obj{"type": "array", "maxItems": 3, "items": obj{"type": "string", "enum": keyed}, "description": "Remove the saved key of these providers."},
+ "clear": obj{"type": "array", "maxItems": len(keyed), "items": obj{"type": "string", "enum": keyed}, "description": "Remove the saved key of these providers."},
"googleEngine": obj{"type": "string", "maxLength": 128, "description": "The Programmable Search Engine ID (cx)."},
"searxngUrl": obj{"type": "string", "maxLength": 2048, "description": "The address of a SearXNG instance."},
},
diff --git a/addons/web/internal/tools/tools.go b/addons/web/internal/tools/tools.go
index bc89f57ad6..32ab0d5263 100644
--- a/addons/web/internal/tools/tools.go
+++ b/addons/web/internal/tools/tools.go
@@ -67,14 +67,14 @@ type Admin interface {
// Endpoints are the addresses of the providers. Tests point them at fakes.
type Endpoints struct {
- Brave, Tavily, Google string
+ Brave, Tavily, Google, Keenable string
// Keyless are the result pages of the keyless chain.
Keyless search.KeylessEndpoints
}
// DefaultEndpoints are the documented addresses.
func DefaultEndpoints() Endpoints {
- return Endpoints{Brave: providers.BraveEndpoint, Tavily: providers.TavilyEndpoint, Google: providers.GoogleEndpoint, Keyless: search.DefaultKeylessEndpoints()}
+ return Endpoints{Brave: providers.BraveEndpoint, Tavily: providers.TavilyEndpoint, Google: providers.GoogleEndpoint, Keenable: providers.KeenableEndpoint, Keyless: search.DefaultKeylessEndpoints()}
}
// Options configure the runtime.
@@ -144,6 +144,8 @@ func (r *runtime) build(s settings.Settings) []search.Provider {
p, err = providers.NewTavily(r.o.Endpoints.Tavily, s.Keys[n])
case providers.Google:
p, err = providers.NewGoogle(r.o.Endpoints.Google, s.Keys[n], s.GoogleEngine)
+ case providers.Keenable:
+ p, err = providers.NewKeenable(r.o.Endpoints.Keenable, s.Keys[n])
case providers.SearXNG:
p, err = search.NewSearXNG(s.SearXNGURL)
case providers.DuckDuckGo:
diff --git a/addons/web/manifest.json b/addons/web/manifest.json
index 9a8e352bf6..33bdadf858 100644
--- a/addons/web/manifest.json
+++ b/addons/web/manifest.json
@@ -108,11 +108,12 @@
"enum": [
"brave",
"tavily",
- "google"
+ "google",
+ "keenable"
],
"type": "string"
},
- "maxItems": 3,
+ "maxItems": 4,
"type": "array"
},
"googleEngine": {
@@ -132,6 +133,10 @@
"maxLength": 256,
"type": "string"
},
+ "keenable": {
+ "maxLength": 256,
+ "type": "string"
+ },
"tavily": {
"maxLength": 256,
"type": "string"
@@ -146,12 +151,13 @@
"brave",
"tavily",
"google",
+ "keenable",
"searxng",
"duckduckgo"
],
"type": "string"
},
- "maxItems": 5,
+ "maxItems": 6,
"type": "array"
},
"searxngUrl": {
diff --git a/addons/web/ui/src/Settings.jsx b/addons/web/ui/src/Settings.jsx
index 03bded72bb..5024b28f1c 100644
--- a/addons/web/ui/src/Settings.jsx
+++ b/addons/web/ui/src/Settings.jsx
@@ -30,7 +30,7 @@ export function createSettings(sdk, t) {
)}
status={{p.ready ? t('web.settings.ready') : t('web.settings.notReady')}}
- meta={!p.ready && p.missing ? p.missing : (p.name === KEYLESS_ENTRY ? t('web.settings.keylessChain') : null)}
+ meta={!p.ready && p.missing ? p.missing : (p.name === KEYLESS_ENTRY ? t('web.settings.keylessChain') : (p.name === 'keenable' && !p.keySet ? t('web.settings.keenableKeyless') : null))}
>
{p.on && (
diff --git a/addons/web/ui/src/index.jsx b/addons/web/ui/src/index.jsx
index 4b2274faa0..ab73a2309d 100644
--- a/addons/web/ui/src/index.jsx
+++ b/addons/web/ui/src/index.jsx
@@ -9,19 +9,21 @@ export function createAddon(sdk) {
const h = React.createElement
sdk.locale.define('en', {
'web.title': 'Web',
- 'web.intro': 'Your chats search the web and read web pages through this add-on, in place of the built-in web tools. It can search with Brave Search, Tavily or Google Programmable Search when an administrator sets a key, and it reads the text of PDF files.',
+ 'web.intro': 'Your chats search the web and read web pages through this add-on, in place of the built-in web tools. It can search with Brave Search, Tavily or Google Programmable Search when an administrator sets a key, or with Keenable, with or without a key, and it reads the text of PDF files.',
'web.rules.order': 'The search providers are tried in the order an administrator chose. When one fails, the next one is tried.',
'web.rules.untrusted': 'What a page says is given to the model as untrusted text. Addresses inside your network are never read.',
'web.rules.settings': 'An administrator sets the providers and their keys on the Settings tab of the Web details, under Add-ons.',
'web.provider.brave': 'Brave Search',
'web.provider.tavily': 'Tavily',
'web.provider.google': 'Google Programmable Search',
+ 'web.provider.keenable': 'Keenable',
'web.provider.searxng': 'SearXNG',
'web.provider.duckduckgo': 'Search without a key',
'web.keyless.duckduckgo': 'DuckDuckGo page',
'web.keyless.brave_html': 'Brave Search page',
'web.keyless.bing_html': 'Bing page',
'web.settings.keylessChain': 'Tries the result pages of DuckDuckGo, Brave Search and Bing in turn, like the built-in web search. It needs no key.',
+ 'web.settings.keenableKeyless': 'Works without a key, limited per IP address. A key lifts the limit.',
'web.settings.cooling': '{{name}} is skipped for a few minutes: {{reason}}.',
'web.settings.loading': 'Reading the web settings…',
'web.settings.intro': 'Choose the search providers of web_search and the order they are tried in. A provider without its key or address is skipped. Keys are kept by the add-on and never shown again.',
diff --git a/addons/web/ui/src/lib/model.js b/addons/web/ui/src/lib/model.js
index 9789e20ffa..ee3a74169d 100644
--- a/addons/web/ui/src/lib/model.js
+++ b/addons/web/ui/src/lib/model.js
@@ -2,8 +2,8 @@
// status tool answers the settings only to an administrator, and never the
// keys: a provider says whether its key is set.
-export const PROVIDERS = ['brave', 'tavily', 'google', 'searxng', 'duckduckgo']
-export const KEYED = ['brave', 'tavily', 'google']
+export const PROVIDERS = ['brave', 'tavily', 'google', 'keenable', 'searxng', 'duckduckgo']
+export const KEYED = ['brave', 'tavily', 'google', 'keenable']
// The keyless entry of the order ('duckduckgo', its saved name) is the chain
// of the built-in search: these result pages, tried in turn.
export const KEYLESS_ENTRY = 'duckduckgo'
diff --git a/addons/web/ui/test/settings.test.mjs b/addons/web/ui/test/settings.test.mjs
index 4792f3e4b1..127d5eec0f 100644
--- a/addons/web/ui/test/settings.test.mjs
+++ b/addons/web/ui/test/settings.test.mjs
@@ -14,6 +14,7 @@ const status = (over = {}) => ({
{ name: 'brave', keyed: true, keySet: true, ready: true },
{ name: 'tavily', keyed: true, keySet: false, ready: false, missing: 'no key' },
{ name: 'google', keyed: true, keySet: false, ready: false, missing: 'no key and no search engine ID' },
+ { name: 'keenable', keyed: true, keySet: false, ready: true },
{ name: 'searxng', keyed: false, keySet: false, ready: false, missing: 'no address' },
{ name: 'duckduckgo', keyed: false, keySet: false, ready: true },
],
@@ -34,7 +35,7 @@ test('settings: the providers of the order come first, in order', async () => {
const { settingsModel, draftOf, rows, move, toggle } = await lib()
const m = settingsModel(status())
let d = draftOf(m)
- assert.deepEqual(rows(m, d).map(r => [r.name, r.on]), [['brave', true], ['duckduckgo', true], ['tavily', false], ['google', false], ['searxng', false]])
+ assert.deepEqual(rows(m, d).map(r => [r.name, r.on]), [['brave', true], ['duckduckgo', true], ['tavily', false], ['google', false], ['keenable', false], ['searxng', false]])
d = move(d, 'duckduckgo', -1)
assert.deepEqual(d.order, ['duckduckgo', 'brave'])
assert.deepEqual(move(d, 'duckduckgo', -1).order, ['duckduckgo', 'brave'], 'the first stays first')