diff --git a/core/application/application.go b/core/application/application.go index b46dd1b86..1c42b57c8 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -1006,7 +1006,7 @@ func (a *Application) startAddons() error { } host := c.Host if host == (contract.HostCompatibility{}) { - host = contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: runtime.GOOS, Arch: runtime.GOARCH} + host = contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: runtime.GOOS, Arch: runtime.GOARCH} } cat = catalog.Catalog{Sources: sources, TargetOS: host.OS, TargetArch: host.Arch} installer = &packages.Installer{Root: filepath.Join(c.Root, "packages"), Catalog: cat, Host: host} diff --git a/core/http/endpoints/addons/gateway.go b/core/http/endpoints/addons/gateway.go index c11d3071a..efbebe37d 100644 --- a/core/http/endpoints/addons/gateway.go +++ b/core/http/endpoints/addons/gateway.go @@ -215,6 +215,8 @@ func operationCauseMessage(code string) string { return "This version of the add-on is installed already. Nothing was installed" case managerpkg.CauseInstalledIncompatible: return "This version of the add-on is not compatible with this server, so it cannot be turned on. Upgrade it to a newer version, then turn it on. Nothing was changed" + case managerpkg.CauseNeedsNewerHost: + return "This add-on needs a newer LocalAI. Update LocalAI, then try again. Nothing was installed" case managerpkg.CauseDowngradeRefused: return "The file holds an older version than the installed one. Confirm the downgrade to install it. Nothing was installed" } diff --git a/core/services/addons/catalog/catalog.go b/core/services/addons/catalog/catalog.go index 6b50de454..a7e25d48d 100644 --- a/core/services/addons/catalog/catalog.go +++ b/core/services/addons/catalog/catalog.go @@ -104,6 +104,12 @@ type Requirements struct { Protocol contract.VersionRange `json:"protocol,omitzero"` DataFormat contract.VersionRange `json:"dataFormat,omitzero"` SDK contract.VersionRange `json:"sdk,omitzero"` + // FeatureLevel is the feature level the package needs (see + // contract.HostFeatureLevel). Zero means no claim, the base level. A host + // older than this field refuses an index that carries it, because the + // index reader is strict; the generator writes it only for a package that + // needs more than the base level. + FeatureLevel uint64 `json:"featureLevel,omitempty"` } type Release struct { diff --git a/core/services/addons/catalog/index_yaml_test.go b/core/services/addons/catalog/index_yaml_test.go index 1c06a12ec..e95e29f73 100644 --- a/core/services/addons/catalog/index_yaml_test.go +++ b/core/services/addons/catalog/index_yaml_test.go @@ -45,6 +45,21 @@ var _ = Describe("library index document (YAML)", func() { Expect(got.Digest).To(Equal(yamlDigest)) Expect(got.UI.MainPage).To(Equal("tables")) }) + It("reads the feature level an entry requires, and an entry without it needs none", func() { + _, e, s := refreshBody(yamlEntry + " requires:\n featureLevel: 2\n") + Expect(e).NotTo(HaveOccurred()) + got, e := (Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}).Resolve(ctx, "lib", "tables", "1.0.0") + Expect(e).NotTo(HaveOccurred()) + Expect(got.Requires.FeatureLevel).To(Equal(uint64(2))) + _, e, s = refreshBody(yamlEntry) + Expect(e).NotTo(HaveOccurred()) + got, e = (Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}).Resolve(ctx, "lib", "tables", "1.0.0") + Expect(e).NotTo(HaveOccurred()) + Expect(got.Requires.FeatureLevel).To(BeZero()) + // The field is bounded like the other ranges. + _, e, _ = refreshBody(yamlEntry + " requires:\n featureLevel: 100000\n") + Expect(e).To(HaveOccurred()) + }) It("accepts an empty library written as an empty list", func() { r, e, _ := refreshBody("[]\n") Expect(e).NotTo(HaveOccurred()) diff --git a/core/services/addons/catalog/sources.go b/core/services/addons/catalog/sources.go index e31d0cd0a..c6e5f9519 100644 --- a/core/services/addons/catalog/sources.go +++ b/core/services/addons/catalog/sources.go @@ -644,6 +644,9 @@ func validTranslations(m map[string]string, max int) bool { // validRequirements accepts a zero range (no claim) or a well formed one. func validRequirements(q Requirements) bool { + if q.FeatureLevel > contract.MaxFeatureLevel { + return false + } for _, r := range []contract.VersionRange{q.Host, q.API, q.Protocol, q.DataFormat, q.SDK} { if r != (contract.VersionRange{}) && (r.Min == 0 || r.Max < r.Min) { return false diff --git a/core/services/addons/gateway/carrier.go b/core/services/addons/gateway/carrier.go index a714f6bbf..13d63308c 100644 --- a/core/services/addons/gateway/carrier.go +++ b/core/services/addons/gateway/carrier.go @@ -645,7 +645,7 @@ func operationCause(e error) string { func IsOperationCause(code string) bool { switch code { case addons.CauseLibraryUnavailable, addons.CauseLibraryConflict, addons.CauseReleaseNotFound, addons.CausePackageFetch, addons.CauseInstallStorage, - addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible: + addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible, addons.CauseNeedsNewerHost: return true } return false @@ -852,7 +852,7 @@ func (c *Carrier) sendCtx(ctx context.Context, a contract.Actor, wire wireReques case wirePackage: return nil, fmt.Errorf("%w: %w", addons.ErrOperationFailed, packages.ErrPackage) case addons.CauseLibraryUnavailable, addons.CauseLibraryConflict, addons.CauseReleaseNotFound, addons.CausePackageFetch, addons.CauseInstallStorage, - addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible: + addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible, addons.CauseNeedsNewerHost: return nil, fmt.Errorf("%w: %w", addons.ErrOperationFailed, &addons.FailureCause{Code: b.Type}) case wireNotPermitted: return nil, ErrNotPermitted diff --git a/core/services/addons/manager.go b/core/services/addons/manager.go index 8107427a3..0269ba1d6 100644 --- a/core/services/addons/manager.go +++ b/core/services/addons/manager.go @@ -89,6 +89,9 @@ const ( // protocol or data format this host no longer runs. It cannot be turned // on; an upgrade to a compatible release can. CauseInstalledIncompatible = "installed_incompatible" + // CauseNeedsNewerHost: the add-on needs a higher feature level than this + // host has (see contract.HostFeatureLevel). + CauseNeedsNewerHost = "needs_newer_host" ) // FailureCause is the typed cause of a failed operation. Its text is fixed and @@ -667,6 +670,10 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl if errors.Is(stageErr, uploads.ErrNotFound) { return fail(stageErr, &FailureCause{Code: CauseFileMissing, detail: stageErr}) } + var level *contract.FeatureLevelRefusal + if errors.As(stageErr, &level) { + return fail(stageErr, &FailureCause{Code: CauseNeedsNewerHost, detail: stageErr}) + } return fail(stageErr, &FailureCause{Code: CausePackageFetch, detail: stageErr}) } if fromFile { diff --git a/core/services/addons/packages/install.go b/core/services/addons/packages/install.go index ad93e1531..2ffae5db4 100644 --- a/core/services/addons/packages/install.go +++ b/core/services/addons/packages/install.go @@ -80,6 +80,11 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func( if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon { return out, ErrPackage } + // The entry says what the package needs. A host that is too old refuses + // before it downloads anything; the manifest is checked again below. + if need := r.Requires.FeatureLevel; need > i.Host.EffectiveFeatureLevel() { + return out, &contract.FeatureLevelRefusal{Need: need, Host: i.Host.EffectiveFeatureLevel()} + } if !r.Equal(trusted) || (r.Digest == "" && !r.Mutable) || r.OS != i.Host.OS || r.Arch != i.Host.Arch { return out, ErrPackage } diff --git a/core/services/addons/packages/install_test.go b/core/services/addons/packages/install_test.go index 5e83095eb..c5a8485ee 100644 --- a/core/services/addons/packages/install_test.go +++ b/core/services/addons/packages/install_test.go @@ -128,6 +128,27 @@ var _ = Describe("immutable packages", func() { Expect(e).To(HaveOccurred()) Expect(filepath.Join(receipt.Directory, "bin/sample")).To(BeAnExistingFile()) }) + It("refuses a package that needs a newer feature level, from the entry and from the manifest", func() { + i, r := fixture(archive(nil, m), func(r *catalog.Release, _ *catalog.Source) { r.Requires.FeatureLevel = 2 }) + _, e := i.Stage(context.Background(), r, nil) + var refusal *contract.FeatureLevelRefusal + Expect(errors.As(e, &refusal)).To(BeTrue()) + Expect(e).To(MatchError("needs a newer LocalAI")) + // The same entry installs on a host that has the level. + i.Host.FeatureLevel = 2 + _, e = i.Stage(context.Background(), r, nil) + Expect(errors.As(e, &refusal)).To(BeFalse()) + + // The manifest decides when the entry says nothing. + needy := bytes.Replace(m, []byte("{"), []byte(`{"featureLevel": 2,`), 1) + i, r = fixture(archive(nil, needy)) + _, e = i.Stage(context.Background(), r, nil) + Expect(errors.As(e, &refusal)).To(BeTrue()) + i.Host.FeatureLevel = 2 + s, e := i.Stage(context.Background(), r, nil) + Expect(e).NotTo(HaveOccurred()) + Expect(i.Discard(s)).To(Succeed()) + }) It("rejects untrusted integrity forged releases and digest mismatches", func() { i, r := fixture(archive(nil, m)) r.Digest = "" diff --git a/core/services/addons/runtime/supervisor.go b/core/services/addons/runtime/supervisor.go index fbc07e599..2546658e2 100644 --- a/core/services/addons/runtime/supervisor.go +++ b/core/services/addons/runtime/supervisor.go @@ -292,7 +292,7 @@ func (s *Supervisor) start(ctx context.Context, pkg packages.PackageReceipt, gen func (s *Supervisor) launch(ctx context.Context, pkg packages.PackageReceipt, generation uint64) (*process, error) { ctx, cancel := context.WithTimeout(ctx, s.config.ReadyTimeout) defer cancel() - host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: goruntime.GOOS, Arch: goruntime.GOARCH} + host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: goruntime.GOOS, Arch: goruntime.GOARCH} if contract.ValidateManifest(pkg.Manifest, host) != nil || !filepath.IsAbs(pkg.Directory) { return nil, ErrRuntime } diff --git a/core/services/addons/updates/checker.go b/core/services/addons/updates/checker.go index 4375ac863..589325e18 100644 --- a/core/services/addons/updates/checker.go +++ b/core/services/addons/updates/checker.go @@ -646,6 +646,9 @@ func Compatibility(host contract.HostCompatibility, installed packages.PackageRe if q.SDK != zero && !accepts(q.SDK, host.SDK) { out = append(out, ReasonSDK) } + if q.FeatureLevel > host.EffectiveFeatureLevel() { + out = append(out, ReasonFeatureLevel) + } // A candidate whose data format ends below the installed one could not // read what the installed package wrote. if q.DataFormat != zero && (!accepts(q.DataFormat, host.DataFormat) || q.DataFormat.Max < installed.Manifest.DataFormat.Max) { diff --git a/core/services/addons/updates/checker_test.go b/core/services/addons/updates/checker_test.go index fac49f86f..d5e8cec7e 100644 --- a/core/services/addons/updates/checker_test.go +++ b/core/services/addons/updates/checker_test.go @@ -351,11 +351,12 @@ var _ = Describe("Update detection", func() { Context("compatibility", func() { It("names the host, SDK and data format ranges a candidate does not fit", func() { for reason, requires := range map[string]catalog.Requirements{ - updates.ReasonHost: {Host: contract.VersionRange{Min: 2, Max: 3}}, - updates.ReasonAPI: {API: contract.VersionRange{Min: 3, Max: 3}}, - updates.ReasonProtocol: {Protocol: contract.VersionRange{Min: 2, Max: 2}}, - updates.ReasonSDK: {SDK: contract.VersionRange{Min: 2, Max: 2}}, - updates.ReasonDataFormat: {DataFormat: contract.VersionRange{Min: 2, Max: 3}}, + updates.ReasonHost: {Host: contract.VersionRange{Min: 2, Max: 3}}, + updates.ReasonAPI: {API: contract.VersionRange{Min: 3, Max: 3}}, + updates.ReasonProtocol: {Protocol: contract.VersionRange{Min: 2, Max: 2}}, + updates.ReasonSDK: {SDK: contract.VersionRange{Min: 2, Max: 2}}, + updates.ReasonDataFormat: {DataFormat: contract.VersionRange{Min: 2, Max: 3}}, + updates.ReasonFeatureLevel: {FeatureLevel: 2}, } { next := rel("1.0.0", "development", digestB) next.Requires = requires diff --git a/core/services/addons/updates/status.go b/core/services/addons/updates/status.go index f45d75ea5..c7be93c58 100644 --- a/core/services/addons/updates/status.go +++ b/core/services/addons/updates/status.go @@ -40,6 +40,8 @@ const ( ReasonProtocol = "protocol-version" ReasonSDK = "sdk-version" ReasonDataFormat = "data-format" + // ReasonFeatureLevel: the release needs a newer LocalAI. + ReasonFeatureLevel = "feature-level" // ReasonDigestChanged is the reason of an IntegrityError. ReasonDigestChanged = "digest-changed" ) diff --git a/core/services/addons/updates_test.go b/core/services/addons/updates_test.go index 2a9aaae85..ed2d0d49b 100644 --- a/core/services/addons/updates_test.go +++ b/core/services/addons/updates_test.go @@ -386,6 +386,19 @@ var _ = Describe("explicit upgrade from an authoritative update check", func() { Expect(s.Generation).To(Equal(uint64(1))) }) + It("refuses a candidate that needs a newer feature level, and offers one the host has", func() { + r := newUpdateRig(true) + s := r.installChannel() + installed := s.Generations[0].Receipt + next := r.lib.publish("1.2.3-dev", "development", "build-2", catalog.Requirements{FeatureLevel: contract.HostFeatureLevel + 1}) + st, e := r.m.CheckUpdates(ctx, actor, "sample") + Expect(e).NotTo(HaveOccurred()) + Expect(st.State).To(Equal(updates.Incompatible)) + Expect(st.Reasons).To(ConsistOf(updates.ReasonFeatureLevel)) + _, e = r.m.Submit(ctx, actor, upgrade("lvl", installed.Digest, next.Digest)) + Expect(e).To(MatchError(ErrIncompatible)) + }) + It("selects a newer immutable version and treats a rewritten version as an integrity error", func() { r := newUpdateRig(true) s := r.installVersion() diff --git a/docs/content/features/addons.md b/docs/content/features/addons.md index 1222746d9..e829c973e 100644 --- a/docs/content/features/addons.md +++ b/docs/content/features/addons.md @@ -1578,6 +1578,36 @@ parent traversal, backslashes, percent escapes or URL schemes. These checks do not substitute for safe extraction, symlink checks, executable mode validation, or checking declared files exist, which belong to package installation. +### Feature level + +The four compatibility ranges above are integers that every published add-on states +(`host`, `api`, `protocol`, `dataFormat`). A new optional manifest field cannot use them +without making every published add-on incompatible, so the host also has a **feature +level**: one integer that counts the additive manifest features it understands. + +- The host declares its level (`contract.HostFeatureLevel`). This version has level 2. + A host that predates the levels is level 1. +- A manifest may set `featureLevel` (a positive integer, default 1). It states the level + the add-on needs. A manifest that uses a field of a higher level must say so: the + parser refuses a manifest that uses `capabilities.resources.agentScoped`, + `capabilities.resources.nounPlural` or `i18n..resources` and does not declare + `featureLevel: 2`. +- A host with a lower level refuses the add-on with the plain reason **needs a newer + LocalAI**. The add-on is not installed and not upgraded. A failed install or upgrade has + the cause `needs_newer_host`, and an update check shows the add-on as incompatible with + the reason `feature-level`. +- The library entry has the optional `requires.featureLevel`. The host reads it before it + downloads the package. If the entry says nothing, the manifest is checked when the + package is staged, with the same reason. + +An older host cannot read the new names. It refuses a manifest with an unknown field, and +it also refuses a **library index** that has an unknown field, because the index reader is +strict too. So one entry with `requires.featureLevel` makes the whole index unreadable +for a host that predates the field. For that reason the `catalog` command of the packager +writes the field only when it gets `--feature-level-in-index`, and the committed index +does not have it. Without it an older host still lists the add-on and refuses it when it +stages the package. Turn the flag on once no host without the field needs the library. + Capabilities are independent: `service` is a boolean, `tools` is a unique list of namespace names, and optional `ui` contains required `entry` and `sdk` range, optional `css`, `pages`, `navigation`, and `chatActions` lists. Navigation names diff --git a/pkg/addons/contract/featurelevel_test.go b/pkg/addons/contract/featurelevel_test.go new file mode 100644 index 000000000..111ae1575 --- /dev/null +++ b/pkg/addons/contract/featurelevel_test.go @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: MIT +package contract + +import ( + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feature level", func() { + base := func() Manifest { + m, e := ParseManifest(fixture()) + Expect(e).NotTo(HaveOccurred()) + return m + } + // scoped is a manifest that uses a level 2 field. + scoped := func() Manifest { + raw, e := os.ReadFile("../../../addons/tables/manifest.json") + Expect(e).NotTo(HaveOccurred()) + m, e := ParseManifest(raw) + Expect(e).NotTo(HaveOccurred()) + m.FeatureLevel = 2 + m.Capabilities.Resources.AgentScoped = true + return m + } + + It("treats a manifest without featureLevel as level 1 and runs it on a level 1 host", func() { + m := base() + Expect(m.RequiredFeatureLevel()).To(Equal(uint64(1))) + h := compatible() + h.FeatureLevel = 1 + Expect(ValidateManifest(m, h)).To(Succeed()) + h.FeatureLevel = 0 // an unset host level is the base level + Expect(ValidateManifest(m, h)).To(Succeed()) + }) + It("refuses an add-on that needs a newer level with a plain reason", func() { + m := scoped() + h := compatible() + h.FeatureLevel = 1 + e := ValidateManifest(m, h) + Expect(e).To(MatchError("needs a newer LocalAI")) + var refusal *FeatureLevelRefusal + Expect(e).To(BeAssignableToTypeOf(refusal)) + h.FeatureLevel = 0 + Expect(ValidateManifest(m, h)).To(MatchError("needs a newer LocalAI")) + }) + It("accepts it on a host that has the level, and the host of this build has it", func() { + Expect(HostFeatureLevel).To(BeNumerically(">=", 2)) + Expect(ValidateManifest(scoped(), compatible())).To(Succeed()) + }) + It("parses a manifest of any level without a host, so the install can say it needs a newer host", func() { + m := base() + m.FeatureLevel = 7 + h := compatible() + Expect(ValidateManifest(m, h)).To(MatchError("needs a newer LocalAI")) + m.FeatureLevel = MaxFeatureLevel + 1 + h.FeatureLevel = MaxFeatureLevel + 5 + Expect(ValidateManifest(m, h)).To(MatchError("invalid featureLevel")) + }) + It("requires the manifest to declare the level of the fields it uses", func() { + m := scoped() + m.FeatureLevel = 0 + Expect(ValidateManifest(m, compatible())).To(MatchError(ContainSubstring("must declare featureLevel 2"))) + m = scoped() + m.FeatureLevel = 1 + Expect(ValidateManifest(m, compatible())).To(MatchError(ContainSubstring("must declare featureLevel 2"))) + m = scoped() + m.Capabilities.Resources.AgentScoped = false + m.Capabilities.Resources.NounPlural = "databases" + m.FeatureLevel = 0 + Expect(ValidateManifest(m, compatible())).NotTo(Succeed(), "a plural noun is a level 2 field") + }) + It("keeps the field out of an older strict reader: an unknown key is refused", func() { + type olderManifest struct { + SchemaVersion int `json:"schemaVersion"` + } + var o olderManifest + Expect(strictJSON([]byte(`{"schemaVersion":1,"featureLevel":2}`), &o, 1024)).NotTo(Succeed()) + var m Manifest + Expect(strictJSON([]byte(`{"schemaVersion":1,"featureLevel":2}`), &m, 1024)).To(Succeed()) + Expect(m.FeatureLevel).To(Equal(uint64(2))) + }) +}) + +var _ = Describe("Agent scoped resources", func() { + load := func() Manifest { + raw, e := os.ReadFile("../../../addons/tables/manifest.json") + Expect(e).NotTo(HaveOccurred()) + m, e := ParseManifest(raw) + Expect(e).NotTo(HaveOccurred()) + m.FeatureLevel = 2 + return m + } + It("needs a list mapping", func() { + m := load() + m.Capabilities.Resources.AgentScoped = true + Expect(ValidateManifest(m, compatible())).To(Succeed()) + m.Capabilities.Resources.List = nil + Expect(ValidateManifest(m, compatible())).To(MatchError(ContainSubstring("need a list mapping"))) + }) + It("accepts a plural and translated nouns, and falls back from the translation to English", func() { + m := load() + r := m.Capabilities.Resources + r.AgentScoped = true + r.Noun, r.NounPlural = "thing", "things" + m.I18n = map[string]Translation{ + "it": {Resources: &ResourceText{Noun: "cosa", NounPlural: "cose"}}, + "fr": {Resources: &ResourceText{Noun: "chose"}}, + } + Expect(ValidateManifest(m, compatible())).To(Succeed()) + for tag, want := range map[string][2]string{ + "": {"thing", "things"}, + "en": {"thing", "things"}, + "it": {"cosa", "cose"}, + "it-IT": {"cosa", "cose"}, + "fr": {"chose", "chose"}, + "de": {"thing", "things"}, + } { + noun, plural := m.ResourceNouns(tag) + Expect([2]string{noun, plural}).To(Equal(want), tag) + } + r.NounPlural = "" + noun, plural := m.ResourceNouns("") + Expect([2]string{noun, plural}).To(Equal([2]string{"thing", "thing"})) + }) + It("refuses translated nouns the manifest does not name, empty ones and markup", func() { + for name, mutate := range map[string]func(*Manifest){ + "no english noun": func(m *Manifest) { m.Capabilities.Resources.Noun = "" }, + "empty": func(m *Manifest) { m.I18n["it"] = Translation{Resources: &ResourceText{}} }, + "markup": func(m *Manifest) { m.I18n["it"] = Translation{Resources: &ResourceText{Noun: "x"}} }, + "plural alone": func(m *Manifest) { m.Capabilities.Resources.Noun, m.Capabilities.Resources.NounPlural = "", "x" }, + } { + m := load() + m.Capabilities.Resources.Noun = "thing" + m.I18n = map[string]Translation{"it": {Resources: &ResourceText{Noun: "cosa"}}} + mutate(&m) + Expect(ValidateManifest(m, compatible())).NotTo(Succeed(), name) + } + }) +}) diff --git a/pkg/addons/contract/i18n.go b/pkg/addons/contract/i18n.go index f56025957..c2b63264c 100644 --- a/pkg/addons/contract/i18n.go +++ b/pkg/addons/contract/i18n.go @@ -31,6 +31,16 @@ const maxAcceptLanguageBytes = 1024 type Translation struct { // Kinds translates connector kinds, by kind ID. Kinds map[string]KindText `json:"kinds,omitempty"` + // Resources translates the noun of the resources (feature level 2). + Resources *ResourceText `json:"resources,omitempty"` +} + +// ResourceText is the translated noun of the resources of an add-on, singular +// and plural. A field left out is shown in the next fallback (see +// Manifest.ResourceNouns). +type ResourceText struct { + Noun string `json:"noun,omitempty"` + NounPlural string `json:"nounPlural,omitempty"` } // KindText is the translated text of one connector kind. A field left out is @@ -68,9 +78,21 @@ func validateI18n(m Manifest) error { return fmt.Errorf("i18n language %q: English is the manifest itself", tag) } seen[key] = true - if len(t.Kinds) == 0 { + if len(t.Kinds) == 0 && t.Resources == nil { return fmt.Errorf("i18n language %q translates nothing", tag) } + if rt := t.Resources; rt != nil { + rs := m.Capabilities.Resources + if rs == nil || rs.Noun == "" || *rt == (ResourceText{}) { + return fmt.Errorf("i18n %q translates resources that the manifest does not name", tag) + } + for _, w := range []string{rt.Noun, rt.NounPlural} { + if w != "" && !plainText(w, 40) { + return fmt.Errorf("i18n %q: invalid resource noun", tag) + } + total += utf8.RuneCountInString(w) + } + } for id, text := range t.Kinds { k, ok := kinds[id] if !ok { @@ -169,6 +191,37 @@ func (m Manifest) KindText(kindID, tag string) (title, description string) { return "", "" } +// ResourceNouns returns the singular and plural noun of the resources for a +// language tag. A translation that gives only the singular is not mixed with +// the English plural: the plural then equals its singular. Without a +// matching translation the English plural is used, else the English noun. +func (m Manifest) ResourceNouns(tag string) (noun, plural string) { + r := m.Capabilities.Resources + if r == nil { + return "", "" + } + noun, plural = r.Noun, r.NounPlural + if plural == "" { + plural = noun + } + langs := make([]string, 0, len(m.I18n)) + for t, v := range m.I18n { + if v.Resources != nil { + langs = append(langs, t) + } + } + if got, ok := MatchLanguage(tag, langs); ok { + t := m.I18n[got].Resources + if t.Noun != "" { + noun, plural = t.Noun, t.Noun + } + if t.NounPlural != "" { + plural = t.NounPlural + } + } + return noun, plural +} + // ValidLanguageTag reports whether s is a language tag a manifest may use: a // BCP 47 subset of a two or three letter language and up to three subtags. func ValidLanguageTag(s string) bool { return len(s) <= 35 && languageTag.MatchString(s) } diff --git a/pkg/addons/contract/manifest.go b/pkg/addons/contract/manifest.go index 215fdf1ca..f93317dd8 100644 --- a/pkg/addons/contract/manifest.go +++ b/pkg/addons/contract/manifest.go @@ -67,14 +67,69 @@ func (r VersionRange) isHostAPI() bool { return r.Min == HostAPIVersion && r.Max == HostAPIVersion } +// HostFeatureLevel is the feature level of this host. The level counts the +// additive manifest features a host understands, apart from the integer +// versions above, so a new optional field does not make every published +// add-on incompatible. A manifest without a featureLevel needs level 1. A host +// older than the first level-2 field reads the field as unknown and refuses +// the manifest, as it refuses any unknown field. +// +// Level 2: capabilities.resources.agentScoped. +const HostFeatureLevel uint64 = 2 + +// BaseFeatureLevel is the level of a manifest that sets no featureLevel. +const BaseFeatureLevel uint64 = 1 + +// MaxFeatureLevel bounds the featureLevel of a manifest. +const MaxFeatureLevel uint64 = 1000 + +// FeatureLevelRefusal is the refusal of an add-on that needs a newer feature +// level than the host has. Its text is meant for people. +type FeatureLevelRefusal struct{ Need, Host uint64 } + +func (e *FeatureLevelRefusal) Error() string { return "needs a newer LocalAI" } + +// EffectiveFeatureLevel is the level a host tuple stands for: an unset level +// is the base level. +func (h HostCompatibility) EffectiveFeatureLevel() uint64 { + if h.FeatureLevel == 0 { + return BaseFeatureLevel + } + return h.FeatureLevel +} + +// RequiredFeatureLevel is the level the manifest declares, 1 when it sets none. +func (m Manifest) RequiredFeatureLevel() uint64 { + if m.FeatureLevel == 0 { + return BaseFeatureLevel + } + return m.FeatureLevel +} + +// usedFeatureLevel is the lowest level that holds every optional field the +// manifest uses. +func (m Manifest) usedFeatureLevel() uint64 { + if r := m.Capabilities.Resources; r != nil && (r.AgentScoped || r.NounPlural != "") { + return 2 + } + for _, t := range m.I18n { + if t.Resources != nil { + return 2 + } + } + return BaseFeatureLevel +} + type HostCompatibility struct { Host uint64 `json:"host"` API uint64 `json:"api"` Protocol uint64 `json:"protocol"` SDK uint64 `json:"sdk"` DataFormat uint64 `json:"dataFormat"` - OS string `json:"os"` - Arch string `json:"arch"` + // FeatureLevel is the feature level of the host; 0 means the base level. + FeatureLevel uint64 `json:"featureLevel,omitempty"` + OS string `json:"os"` + Arch string `json:"arch"` } type Manifest struct { SchemaVersion int `json:"schemaVersion"` @@ -87,6 +142,12 @@ type Manifest struct { API VersionRange `json:"api"` Protocol VersionRange `json:"protocol"` DataFormat VersionRange `json:"dataFormat"` + // FeatureLevel is the feature level the manifest needs from the host. It is + // optional and defaults to 1. A manifest that uses a field of a higher + // level must say so; a host with a lower level refuses it with a plain + // reason. A host that predates the field refuses the manifest as it + // refuses any unknown field. + FeatureLevel uint64 `json:"featureLevel,omitempty"` // LaunchMode is absent or "always": the owner starts the addon while it is // enabled. "on-demand" is reserved and rejected until it is implemented. LaunchMode string `json:"launchMode,omitempty"` @@ -157,6 +218,16 @@ type Capabilities struct { type ResourceTools struct { // Noun is the singular word for one resource in the UI, such as database. Noun string `json:"noun,omitempty"` + // NounPlural is the plural of Noun, such as databases. It is optional and + // needs feature level 2; without it the UI uses the noun alone. A + // translation of both words is under i18n..resources. + NounPlural string `json:"nounPlural,omitempty"` + // AgentScoped opts the add-on in to per-agent resource lists: a saved + // agent then names the resources it may use and the mode of each, and the + // host enforces the list on every tool call that takes a resource. It + // needs feature level 2 and a List mapping, because the host builds the + // choices from the list tool. + AgentScoped bool `json:"agentScoped,omitempty"` // Create is a tool of Tools whose definition has the kind create. Create string `json:"create,omitempty"` // Rename is a tool of Tools whose definition has the kind schema and takes @@ -213,6 +284,12 @@ func validateResourceTools(c Capabilities) error { if r.Noun != "" && !plainText(r.Noun, 40) { return fmt.Errorf("invalid resource noun") } + if r.NounPlural != "" && (r.Noun == "" || !plainText(r.NounPlural, 40)) { + return fmt.Errorf("invalid resource plural noun") + } + if r.AgentScoped && r.List == nil { + return fmt.Errorf("agentScoped resources need a list mapping") + } for _, want := range []struct{ name, kind, label string }{ {r.Create, ToolKindCreate, "create"}, {r.Rename, ToolKindSchema, "rename"}, {r.Inspect, ToolKindRead, "inspect"}, } { @@ -363,7 +440,7 @@ func ParseManifest(data []byte) (Manifest, error) { m.Capabilities.ToolDefinitions[i].InputSchema = c } // Parsing validates intrinsic constraints without imposing this machine's platform. - h := HostCompatibility{Host: m.Host.Min, API: m.API.Min, Protocol: m.Protocol.Min, DataFormat: m.DataFormat.Min, OS: m.OS, Arch: m.Arch} + h := HostCompatibility{Host: m.Host.Min, API: m.API.Min, Protocol: m.Protocol.Min, DataFormat: m.DataFormat.Min, FeatureLevel: m.RequiredFeatureLevel(), OS: m.OS, Arch: m.Arch} if m.Capabilities.UI != nil { h.SDK = m.Capabilities.UI.SDK.Min } @@ -390,6 +467,15 @@ func ValidateManifest(m Manifest, h HostCompatibility) error { if !m.Host.accepts(h.Host) || !m.API.accepts(h.API) || !m.Protocol.accepts(h.Protocol) || !m.DataFormat.accepts(h.DataFormat) { return fmt.Errorf("incompatible host, API, protocol or data format") } + if m.FeatureLevel > MaxFeatureLevel { + return fmt.Errorf("invalid featureLevel") + } + if used := m.usedFeatureLevel(); m.RequiredFeatureLevel() < used { + return fmt.Errorf("the manifest uses a field of feature level %d and must declare featureLevel %d", used, used) + } + if need := m.RequiredFeatureLevel(); need > h.EffectiveFeatureLevel() { + return &FeatureLevelRefusal{Need: need, Host: h.EffectiveFeatureLevel()} + } c := m.Capabilities if (!c.Service && len(c.Tools) == 0 && c.UI == nil && c.Connector == nil) || !uniqueIDs(c.Tools) { return fmt.Errorf("invalid capabilities") diff --git a/pkg/addons/contract/manifest_test.go b/pkg/addons/contract/manifest_test.go index 4f093e9cb..0de674f62 100644 --- a/pkg/addons/contract/manifest_test.go +++ b/pkg/addons/contract/manifest_test.go @@ -16,7 +16,7 @@ func fixture() []byte { return b } func compatible() HostCompatibility { - return HostCompatibility{Host: 1, API: 2, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"} + return HostCompatibility{Host: 1, API: 2, Protocol: 1, SDK: 1, DataFormat: 1, FeatureLevel: HostFeatureLevel, OS: "linux", Arch: "amd64"} } var _ = Describe("Manifest", func() { diff --git a/pkg/addons/packaging/build.go b/pkg/addons/packaging/build.go index 212138a56..dc6ca7986 100644 --- a/pkg/addons/packaging/build.go +++ b/pkg/addons/packaging/build.go @@ -32,7 +32,7 @@ import ( // does not contain them is not installable and must not be published. func hostFor(platform string) contract.HostCompatibility { os_, arch, _ := strings.Cut(platform, "/") - return contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: os_, Arch: arch} + return contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: os_, Arch: arch} } type fileEntry struct { diff --git a/pkg/addons/packaging/catalog.go b/pkg/addons/packaging/catalog.go index 8ea686201..1d453424d 100644 --- a/pkg/addons/packaging/catalog.go +++ b/pkg/addons/packaging/catalog.go @@ -50,7 +50,13 @@ type catalogRelease struct { // SkillShadows repeats catalog.Release.SkillShadows: the built-in skills // the manifest's skill definitions replace. SkillShadows []string `json:"skillShadows,omitempty"` - UI struct { + // Requires repeats catalog.Release.Requires for the one field the + // generator writes: the feature level of a package that needs more than + // the base level. + Requires struct { + FeatureLevel uint64 `json:"featureLevel,omitempty"` + } `json:"requires,omitzero"` + UI struct { Title string `json:"title"` Description string `json:"description"` Titles map[string]string `json:"titles,omitempty"` @@ -140,6 +146,19 @@ func translationFieldsYAML(ui *contract.UI, indent string) string { return b.String() } +// requiresFieldsYAML renders the requires block of a library entry: the +// feature level of a manifest that needs more than the base level. A manifest +// at the base level has no block. The index reader of a host that predates +// the field refuses the whole index when one entry has it (the reader is +// strict), so the catalog command writes it only with +// --feature-level-in-index. +func requiresFieldsYAML(m contract.Manifest, indent string) string { + if m.RequiredFeatureLevel() <= contract.BaseFeatureLevel { + return "" + } + return fmt.Sprintf("%srequires:\n%s featureLevel: %d\n", indent, indent, m.RequiredFeatureLevel()) +} + // cmdCatalogTools adds the toolCount and tools fields to the entries of an // existing library index, reading each manifest from addons//manifest.json // of the repository. It touches no other field, so the digests and artifacts @@ -231,6 +250,7 @@ func cmdCatalog(args []string) error { out := fs.String("out", "", "output file") root := fs.String("root", ".", "with --from-manifest: repository root that the manifest path of the platform matrix is relative to") fromManifest := fs.Bool("from-manifest", false, "development channel only: render from the manifest and the enabled platforms, without a built package") + featureLevelInIndex := fs.Bool("feature-level-in-index", false, "write requires.featureLevel for an add-on that needs more than level 1. A host that predates the field refuses the whole index, so leave it off while such hosts are in use") if err := fs.Parse(args); err != nil { return err } @@ -297,6 +317,9 @@ func cmdCatalog(args []string) error { // text lands inside it. text = strings.TrimRight(text, "\n") + "\n" + translationFieldsYAML(ui, " ") + toolFieldsYAML(manifest.Capabilities.Tools, "", contract.ShadowedTools(manifest.Capabilities.ToolDefinitions)...) + skillFieldsYAML(contract.ShadowedSkills(manifest.Capabilities.SkillDefinitions), "") + if *featureLevelInIndex { + text += requiresFieldsYAML(*manifest, "") + } if mutable { text = strings.Replace(text, "digest: \"{{digest}}\"", "mutable: true", 1) } diff --git a/pkg/addons/packaging/featurelevel_test.go b/pkg/addons/packaging/featurelevel_test.go new file mode 100644 index 000000000..fb5da446f --- /dev/null +++ b/pkg/addons/packaging/featurelevel_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT +package packaging + +import ( + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("feature level in the library index", func() { + render := func(id string, edit func(string) string, extra ...string) []catalogRelease { + GinkgoHelper() + p := newProjectFor(id, both) + path := filepath.Join(p.root, "addons", id, "manifest.json") + b, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(path, []byte(edit(string(b))), 0o644)).To(Succeed()) + out := filepath.Join(GinkgoT().TempDir(), "catalog.yaml") + args := append([]string{"catalog", "--config", p.config, "--root", p.root, "--template", "../../../gallery/addons/" + id + ".yaml.tmpl", "--from-manifest", "--repository", newRegistry().repo("addons"), "--channel", "development", "--owner-repo", "example/localai", "--signer-ref", "refs/heads/master", "--out", out}, extra...) + _, err = run(args...) + Expect(err).NotTo(HaveOccurred(), id) + raw, err := os.ReadFile(out) + Expect(err).NotTo(HaveOccurred()) + entries, err := readCatalogIndex(raw) + Expect(err).NotTo(HaveOccurred(), id) + Expect(entries).NotTo(BeEmpty()) + return entries + } + needsTwo := func(s string) string { + return strings.Replace(s, `"schemaVersion": 1,`, `"schemaVersion": 1, "featureLevel": 2,`, 1) + } + It("writes nothing for a manifest at the base level, with or without the flag", func() { + for _, extra := range [][]string{nil, {"--feature-level-in-index"}} { + for _, e := range render("telegram", func(s string) string { return s }, extra...) { + Expect(e.Requires.FeatureLevel).To(BeZero()) + } + } + }) + It("keeps the level out of the index by default, so a host that predates the field still reads it", func() { + for _, e := range render("telegram", needsTwo) { + Expect(e.Requires.FeatureLevel).To(BeZero()) + } + }) + It("writes the level the manifest needs when asked to", func() { + for _, e := range render("telegram", needsTwo, "--feature-level-in-index") { + Expect(e.Requires.FeatureLevel).To(Equal(uint64(2))) + } + }) +}) diff --git a/pkg/addons/providerconformance/conformance.go b/pkg/addons/providerconformance/conformance.go index c84b0ec4f..0bbbaeecc 100644 --- a/pkg/addons/providerconformance/conformance.go +++ b/pkg/addons/providerconformance/conformance.go @@ -174,7 +174,7 @@ func checkManifest(t *testing.T, s Subject) (contract.Manifest, contract.Provide if m, err = contract.ParseManifest(s.Manifest); err != nil { t.Fatalf("manifest: %v", err) } - host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: m.OS, Arch: m.Arch} + host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: m.OS, Arch: m.Arch} if err := contract.ValidateManifest(m, host); err != nil { t.Fatalf("the host refuses the manifest: %v", err) }