mirror of
https://github.com/mudler/LocalAI.git
synced 2026-11-05 12:03:27 -05:00
feat(addons): add a feature level for new manifest fields
The manifest parser refuses unknown fields and every published add-on
says host {1,1}, so a new optional field could only be gated by raising
the host number, which would refuse all of them. A separate feature
level does the same job.
The host declares its level (2 in this version). A manifest may set
featureLevel (default 1), and must set it when it uses a field of a
higher level: resources.agentScoped, resources.nounPlural and the
translated resource nouns in i18n are level 2. A host with a lower level
refuses the add-on with "needs a newer LocalAI". The update check shows
it as incompatible (reason feature-level), a library entry with
requires.featureLevel is refused before the download, and a failed
install has the cause needs_newer_host.
The index reader is strict: a host that predates the new requires field
refuses the whole index when one entry has it. The catalog command
therefore writes the field only with --feature-level-in-index, and the
committed index does not use it. The manifest is checked when the
package is staged either way.
Assisted-by: Claude:claude-sonnet-5-5
This commit is contained in:
23 files changed
+480
-17
No files matched your search
@@ -1006,7 +1006,7 @@ func (a *Application) startAddons() error {
|
||||
}
|
||||
host := c.Host
|
||||
if host == (contract.HostCompatibility{}) {
|
||||
host = contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: runtime.GOOS, Arch: runtime.GOARCH}
|
||||
host = contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: runtime.GOOS, Arch: runtime.GOARCH}
|
||||
}
|
||||
cat = catalog.Catalog{Sources: sources, TargetOS: host.OS, TargetArch: host.Arch}
|
||||
installer = &packages.Installer{Root: filepath.Join(c.Root, "packages"), Catalog: cat, Host: host}
|
||||
|
||||
@@ -215,6 +215,8 @@ func operationCauseMessage(code string) string {
|
||||
return "This version of the add-on is installed already. Nothing was installed"
|
||||
case managerpkg.CauseInstalledIncompatible:
|
||||
return "This version of the add-on is not compatible with this server, so it cannot be turned on. Upgrade it to a newer version, then turn it on. Nothing was changed"
|
||||
case managerpkg.CauseNeedsNewerHost:
|
||||
return "This add-on needs a newer LocalAI. Update LocalAI, then try again. Nothing was installed"
|
||||
case managerpkg.CauseDowngradeRefused:
|
||||
return "The file holds an older version than the installed one. Confirm the downgrade to install it. Nothing was installed"
|
||||
}
|
||||
|
||||
@@ -104,6 +104,12 @@ type Requirements struct {
|
||||
Protocol contract.VersionRange `json:"protocol,omitzero"`
|
||||
DataFormat contract.VersionRange `json:"dataFormat,omitzero"`
|
||||
SDK contract.VersionRange `json:"sdk,omitzero"`
|
||||
// FeatureLevel is the feature level the package needs (see
|
||||
// contract.HostFeatureLevel). Zero means no claim, the base level. A host
|
||||
// older than this field refuses an index that carries it, because the
|
||||
// index reader is strict; the generator writes it only for a package that
|
||||
// needs more than the base level.
|
||||
FeatureLevel uint64 `json:"featureLevel,omitempty"`
|
||||
}
|
||||
|
||||
type Release struct {
|
||||
|
||||
@@ -45,6 +45,21 @@ var _ = Describe("library index document (YAML)", func() {
|
||||
Expect(got.Digest).To(Equal(yamlDigest))
|
||||
Expect(got.UI.MainPage).To(Equal("tables"))
|
||||
})
|
||||
It("reads the feature level an entry requires, and an entry without it needs none", func() {
|
||||
_, e, s := refreshBody(yamlEntry + " requires:\n featureLevel: 2\n")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
got, e := (Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}).Resolve(ctx, "lib", "tables", "1.0.0")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(got.Requires.FeatureLevel).To(Equal(uint64(2)))
|
||||
_, e, s = refreshBody(yamlEntry)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
got, e = (Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}).Resolve(ctx, "lib", "tables", "1.0.0")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(got.Requires.FeatureLevel).To(BeZero())
|
||||
// The field is bounded like the other ranges.
|
||||
_, e, _ = refreshBody(yamlEntry + " requires:\n featureLevel: 100000\n")
|
||||
Expect(e).To(HaveOccurred())
|
||||
})
|
||||
It("accepts an empty library written as an empty list", func() {
|
||||
r, e, _ := refreshBody("[]\n")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
|
||||
@@ -644,6 +644,9 @@ func validTranslations(m map[string]string, max int) bool {
|
||||
|
||||
// validRequirements accepts a zero range (no claim) or a well formed one.
|
||||
func validRequirements(q Requirements) bool {
|
||||
if q.FeatureLevel > contract.MaxFeatureLevel {
|
||||
return false
|
||||
}
|
||||
for _, r := range []contract.VersionRange{q.Host, q.API, q.Protocol, q.DataFormat, q.SDK} {
|
||||
if r != (contract.VersionRange{}) && (r.Min == 0 || r.Max < r.Min) {
|
||||
return false
|
||||
|
||||
@@ -645,7 +645,7 @@ func operationCause(e error) string {
|
||||
func IsOperationCause(code string) bool {
|
||||
switch code {
|
||||
case addons.CauseLibraryUnavailable, addons.CauseLibraryConflict, addons.CauseReleaseNotFound, addons.CausePackageFetch, addons.CauseInstallStorage,
|
||||
addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible:
|
||||
addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible, addons.CauseNeedsNewerHost:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -852,7 +852,7 @@ func (c *Carrier) sendCtx(ctx context.Context, a contract.Actor, wire wireReques
|
||||
case wirePackage:
|
||||
return nil, fmt.Errorf("%w: %w", addons.ErrOperationFailed, packages.ErrPackage)
|
||||
case addons.CauseLibraryUnavailable, addons.CauseLibraryConflict, addons.CauseReleaseNotFound, addons.CausePackageFetch, addons.CauseInstallStorage,
|
||||
addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible:
|
||||
addons.CauseFileMissing, addons.CauseVersionNotNewer, addons.CauseDowngradeRefused, addons.CauseInstalledIncompatible, addons.CauseNeedsNewerHost:
|
||||
return nil, fmt.Errorf("%w: %w", addons.ErrOperationFailed, &addons.FailureCause{Code: b.Type})
|
||||
case wireNotPermitted:
|
||||
return nil, ErrNotPermitted
|
||||
|
||||
@@ -89,6 +89,9 @@ const (
|
||||
// protocol or data format this host no longer runs. It cannot be turned
|
||||
// on; an upgrade to a compatible release can.
|
||||
CauseInstalledIncompatible = "installed_incompatible"
|
||||
// CauseNeedsNewerHost: the add-on needs a higher feature level than this
|
||||
// host has (see contract.HostFeatureLevel).
|
||||
CauseNeedsNewerHost = "needs_newer_host"
|
||||
)
|
||||
|
||||
// FailureCause is the typed cause of a failed operation. Its text is fixed and
|
||||
@@ -667,6 +670,10 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl
|
||||
if errors.Is(stageErr, uploads.ErrNotFound) {
|
||||
return fail(stageErr, &FailureCause{Code: CauseFileMissing, detail: stageErr})
|
||||
}
|
||||
var level *contract.FeatureLevelRefusal
|
||||
if errors.As(stageErr, &level) {
|
||||
return fail(stageErr, &FailureCause{Code: CauseNeedsNewerHost, detail: stageErr})
|
||||
}
|
||||
return fail(stageErr, &FailureCause{Code: CausePackageFetch, detail: stageErr})
|
||||
}
|
||||
if fromFile {
|
||||
|
||||
@@ -80,6 +80,11 @@ func (i *Installer) Stage(ctx context.Context, r catalog.Release, progress func(
|
||||
if r.Policy.IdentityRegex != "" || r.EntryKind() != catalog.KindAddon {
|
||||
return out, ErrPackage
|
||||
}
|
||||
// The entry says what the package needs. A host that is too old refuses
|
||||
// before it downloads anything; the manifest is checked again below.
|
||||
if need := r.Requires.FeatureLevel; need > i.Host.EffectiveFeatureLevel() {
|
||||
return out, &contract.FeatureLevelRefusal{Need: need, Host: i.Host.EffectiveFeatureLevel()}
|
||||
}
|
||||
if !r.Equal(trusted) || (r.Digest == "" && !r.Mutable) || r.OS != i.Host.OS || r.Arch != i.Host.Arch {
|
||||
return out, ErrPackage
|
||||
}
|
||||
|
||||
@@ -128,6 +128,27 @@ var _ = Describe("immutable packages", func() {
|
||||
Expect(e).To(HaveOccurred())
|
||||
Expect(filepath.Join(receipt.Directory, "bin/sample")).To(BeAnExistingFile())
|
||||
})
|
||||
It("refuses a package that needs a newer feature level, from the entry and from the manifest", func() {
|
||||
i, r := fixture(archive(nil, m), func(r *catalog.Release, _ *catalog.Source) { r.Requires.FeatureLevel = 2 })
|
||||
_, e := i.Stage(context.Background(), r, nil)
|
||||
var refusal *contract.FeatureLevelRefusal
|
||||
Expect(errors.As(e, &refusal)).To(BeTrue())
|
||||
Expect(e).To(MatchError("needs a newer LocalAI"))
|
||||
// The same entry installs on a host that has the level.
|
||||
i.Host.FeatureLevel = 2
|
||||
_, e = i.Stage(context.Background(), r, nil)
|
||||
Expect(errors.As(e, &refusal)).To(BeFalse())
|
||||
|
||||
// The manifest decides when the entry says nothing.
|
||||
needy := bytes.Replace(m, []byte("{"), []byte(`{"featureLevel": 2,`), 1)
|
||||
i, r = fixture(archive(nil, needy))
|
||||
_, e = i.Stage(context.Background(), r, nil)
|
||||
Expect(errors.As(e, &refusal)).To(BeTrue())
|
||||
i.Host.FeatureLevel = 2
|
||||
s, e := i.Stage(context.Background(), r, nil)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(i.Discard(s)).To(Succeed())
|
||||
})
|
||||
It("rejects untrusted integrity forged releases and digest mismatches", func() {
|
||||
i, r := fixture(archive(nil, m))
|
||||
r.Digest = ""
|
||||
|
||||
@@ -292,7 +292,7 @@ func (s *Supervisor) start(ctx context.Context, pkg packages.PackageReceipt, gen
|
||||
func (s *Supervisor) launch(ctx context.Context, pkg packages.PackageReceipt, generation uint64) (*process, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.config.ReadyTimeout)
|
||||
defer cancel()
|
||||
host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: goruntime.GOOS, Arch: goruntime.GOARCH}
|
||||
host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: goruntime.GOOS, Arch: goruntime.GOARCH}
|
||||
if contract.ValidateManifest(pkg.Manifest, host) != nil || !filepath.IsAbs(pkg.Directory) {
|
||||
return nil, ErrRuntime
|
||||
}
|
||||
|
||||
@@ -646,6 +646,9 @@ func Compatibility(host contract.HostCompatibility, installed packages.PackageRe
|
||||
if q.SDK != zero && !accepts(q.SDK, host.SDK) {
|
||||
out = append(out, ReasonSDK)
|
||||
}
|
||||
if q.FeatureLevel > host.EffectiveFeatureLevel() {
|
||||
out = append(out, ReasonFeatureLevel)
|
||||
}
|
||||
// A candidate whose data format ends below the installed one could not
|
||||
// read what the installed package wrote.
|
||||
if q.DataFormat != zero && (!accepts(q.DataFormat, host.DataFormat) || q.DataFormat.Max < installed.Manifest.DataFormat.Max) {
|
||||
|
||||
@@ -351,11 +351,12 @@ var _ = Describe("Update detection", func() {
|
||||
Context("compatibility", func() {
|
||||
It("names the host, SDK and data format ranges a candidate does not fit", func() {
|
||||
for reason, requires := range map[string]catalog.Requirements{
|
||||
updates.ReasonHost: {Host: contract.VersionRange{Min: 2, Max: 3}},
|
||||
updates.ReasonAPI: {API: contract.VersionRange{Min: 3, Max: 3}},
|
||||
updates.ReasonProtocol: {Protocol: contract.VersionRange{Min: 2, Max: 2}},
|
||||
updates.ReasonSDK: {SDK: contract.VersionRange{Min: 2, Max: 2}},
|
||||
updates.ReasonDataFormat: {DataFormat: contract.VersionRange{Min: 2, Max: 3}},
|
||||
updates.ReasonHost: {Host: contract.VersionRange{Min: 2, Max: 3}},
|
||||
updates.ReasonAPI: {API: contract.VersionRange{Min: 3, Max: 3}},
|
||||
updates.ReasonProtocol: {Protocol: contract.VersionRange{Min: 2, Max: 2}},
|
||||
updates.ReasonSDK: {SDK: contract.VersionRange{Min: 2, Max: 2}},
|
||||
updates.ReasonDataFormat: {DataFormat: contract.VersionRange{Min: 2, Max: 3}},
|
||||
updates.ReasonFeatureLevel: {FeatureLevel: 2},
|
||||
} {
|
||||
next := rel("1.0.0", "development", digestB)
|
||||
next.Requires = requires
|
||||
|
||||
@@ -40,6 +40,8 @@ const (
|
||||
ReasonProtocol = "protocol-version"
|
||||
ReasonSDK = "sdk-version"
|
||||
ReasonDataFormat = "data-format"
|
||||
// ReasonFeatureLevel: the release needs a newer LocalAI.
|
||||
ReasonFeatureLevel = "feature-level"
|
||||
// ReasonDigestChanged is the reason of an IntegrityError.
|
||||
ReasonDigestChanged = "digest-changed"
|
||||
)
|
||||
|
||||
@@ -386,6 +386,19 @@ var _ = Describe("explicit upgrade from an authoritative update check", func() {
|
||||
Expect(s.Generation).To(Equal(uint64(1)))
|
||||
})
|
||||
|
||||
It("refuses a candidate that needs a newer feature level, and offers one the host has", func() {
|
||||
r := newUpdateRig(true)
|
||||
s := r.installChannel()
|
||||
installed := s.Generations[0].Receipt
|
||||
next := r.lib.publish("1.2.3-dev", "development", "build-2", catalog.Requirements{FeatureLevel: contract.HostFeatureLevel + 1})
|
||||
st, e := r.m.CheckUpdates(ctx, actor, "sample")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(st.State).To(Equal(updates.Incompatible))
|
||||
Expect(st.Reasons).To(ConsistOf(updates.ReasonFeatureLevel))
|
||||
_, e = r.m.Submit(ctx, actor, upgrade("lvl", installed.Digest, next.Digest))
|
||||
Expect(e).To(MatchError(ErrIncompatible))
|
||||
})
|
||||
|
||||
It("selects a newer immutable version and treats a rewritten version as an integrity error", func() {
|
||||
r := newUpdateRig(true)
|
||||
s := r.installVersion()
|
||||
|
||||
@@ -1578,6 +1578,36 @@ parent traversal, backslashes, percent escapes or URL schemes. These checks do
|
||||
not substitute for safe extraction, symlink checks, executable mode validation,
|
||||
or checking declared files exist, which belong to package installation.
|
||||
|
||||
### Feature level
|
||||
|
||||
The four compatibility ranges above are integers that every published add-on states
|
||||
(`host`, `api`, `protocol`, `dataFormat`). A new optional manifest field cannot use them
|
||||
without making every published add-on incompatible, so the host also has a **feature
|
||||
level**: one integer that counts the additive manifest features it understands.
|
||||
|
||||
- The host declares its level (`contract.HostFeatureLevel`). This version has level 2.
|
||||
A host that predates the levels is level 1.
|
||||
- A manifest may set `featureLevel` (a positive integer, default 1). It states the level
|
||||
the add-on needs. A manifest that uses a field of a higher level must say so: the
|
||||
parser refuses a manifest that uses `capabilities.resources.agentScoped`,
|
||||
`capabilities.resources.nounPlural` or `i18n.<language>.resources` and does not declare
|
||||
`featureLevel: 2`.
|
||||
- A host with a lower level refuses the add-on with the plain reason **needs a newer
|
||||
LocalAI**. The add-on is not installed and not upgraded. A failed install or upgrade has
|
||||
the cause `needs_newer_host`, and an update check shows the add-on as incompatible with
|
||||
the reason `feature-level`.
|
||||
- The library entry has the optional `requires.featureLevel`. The host reads it before it
|
||||
downloads the package. If the entry says nothing, the manifest is checked when the
|
||||
package is staged, with the same reason.
|
||||
|
||||
An older host cannot read the new names. It refuses a manifest with an unknown field, and
|
||||
it also refuses a **library index** that has an unknown field, because the index reader is
|
||||
strict too. So one entry with `requires.featureLevel` makes the whole index unreadable
|
||||
for a host that predates the field. For that reason the `catalog` command of the packager
|
||||
writes the field only when it gets `--feature-level-in-index`, and the committed index
|
||||
does not have it. Without it an older host still lists the add-on and refuses it when it
|
||||
stages the package. Turn the flag on once no host without the field needs the library.
|
||||
|
||||
Capabilities are independent: `service` is a boolean, `tools` is a unique list
|
||||
of namespace names, and optional `ui` contains required `entry` and `sdk` range,
|
||||
optional `css`, `pages`, `navigation`, and `chatActions` lists. Navigation names
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package contract
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Feature level", func() {
|
||||
base := func() Manifest {
|
||||
m, e := ParseManifest(fixture())
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
return m
|
||||
}
|
||||
// scoped is a manifest that uses a level 2 field.
|
||||
scoped := func() Manifest {
|
||||
raw, e := os.ReadFile("../../../addons/tables/manifest.json")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
m, e := ParseManifest(raw)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
m.FeatureLevel = 2
|
||||
m.Capabilities.Resources.AgentScoped = true
|
||||
return m
|
||||
}
|
||||
|
||||
It("treats a manifest without featureLevel as level 1 and runs it on a level 1 host", func() {
|
||||
m := base()
|
||||
Expect(m.RequiredFeatureLevel()).To(Equal(uint64(1)))
|
||||
h := compatible()
|
||||
h.FeatureLevel = 1
|
||||
Expect(ValidateManifest(m, h)).To(Succeed())
|
||||
h.FeatureLevel = 0 // an unset host level is the base level
|
||||
Expect(ValidateManifest(m, h)).To(Succeed())
|
||||
})
|
||||
It("refuses an add-on that needs a newer level with a plain reason", func() {
|
||||
m := scoped()
|
||||
h := compatible()
|
||||
h.FeatureLevel = 1
|
||||
e := ValidateManifest(m, h)
|
||||
Expect(e).To(MatchError("needs a newer LocalAI"))
|
||||
var refusal *FeatureLevelRefusal
|
||||
Expect(e).To(BeAssignableToTypeOf(refusal))
|
||||
h.FeatureLevel = 0
|
||||
Expect(ValidateManifest(m, h)).To(MatchError("needs a newer LocalAI"))
|
||||
})
|
||||
It("accepts it on a host that has the level, and the host of this build has it", func() {
|
||||
Expect(HostFeatureLevel).To(BeNumerically(">=", 2))
|
||||
Expect(ValidateManifest(scoped(), compatible())).To(Succeed())
|
||||
})
|
||||
It("parses a manifest of any level without a host, so the install can say it needs a newer host", func() {
|
||||
m := base()
|
||||
m.FeatureLevel = 7
|
||||
h := compatible()
|
||||
Expect(ValidateManifest(m, h)).To(MatchError("needs a newer LocalAI"))
|
||||
m.FeatureLevel = MaxFeatureLevel + 1
|
||||
h.FeatureLevel = MaxFeatureLevel + 5
|
||||
Expect(ValidateManifest(m, h)).To(MatchError("invalid featureLevel"))
|
||||
})
|
||||
It("requires the manifest to declare the level of the fields it uses", func() {
|
||||
m := scoped()
|
||||
m.FeatureLevel = 0
|
||||
Expect(ValidateManifest(m, compatible())).To(MatchError(ContainSubstring("must declare featureLevel 2")))
|
||||
m = scoped()
|
||||
m.FeatureLevel = 1
|
||||
Expect(ValidateManifest(m, compatible())).To(MatchError(ContainSubstring("must declare featureLevel 2")))
|
||||
m = scoped()
|
||||
m.Capabilities.Resources.AgentScoped = false
|
||||
m.Capabilities.Resources.NounPlural = "databases"
|
||||
m.FeatureLevel = 0
|
||||
Expect(ValidateManifest(m, compatible())).NotTo(Succeed(), "a plural noun is a level 2 field")
|
||||
})
|
||||
It("keeps the field out of an older strict reader: an unknown key is refused", func() {
|
||||
type olderManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
}
|
||||
var o olderManifest
|
||||
Expect(strictJSON([]byte(`{"schemaVersion":1,"featureLevel":2}`), &o, 1024)).NotTo(Succeed())
|
||||
var m Manifest
|
||||
Expect(strictJSON([]byte(`{"schemaVersion":1,"featureLevel":2}`), &m, 1024)).To(Succeed())
|
||||
Expect(m.FeatureLevel).To(Equal(uint64(2)))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("Agent scoped resources", func() {
|
||||
load := func() Manifest {
|
||||
raw, e := os.ReadFile("../../../addons/tables/manifest.json")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
m, e := ParseManifest(raw)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
m.FeatureLevel = 2
|
||||
return m
|
||||
}
|
||||
It("needs a list mapping", func() {
|
||||
m := load()
|
||||
m.Capabilities.Resources.AgentScoped = true
|
||||
Expect(ValidateManifest(m, compatible())).To(Succeed())
|
||||
m.Capabilities.Resources.List = nil
|
||||
Expect(ValidateManifest(m, compatible())).To(MatchError(ContainSubstring("need a list mapping")))
|
||||
})
|
||||
It("accepts a plural and translated nouns, and falls back from the translation to English", func() {
|
||||
m := load()
|
||||
r := m.Capabilities.Resources
|
||||
r.AgentScoped = true
|
||||
r.Noun, r.NounPlural = "thing", "things"
|
||||
m.I18n = map[string]Translation{
|
||||
"it": {Resources: &ResourceText{Noun: "cosa", NounPlural: "cose"}},
|
||||
"fr": {Resources: &ResourceText{Noun: "chose"}},
|
||||
}
|
||||
Expect(ValidateManifest(m, compatible())).To(Succeed())
|
||||
for tag, want := range map[string][2]string{
|
||||
"": {"thing", "things"},
|
||||
"en": {"thing", "things"},
|
||||
"it": {"cosa", "cose"},
|
||||
"it-IT": {"cosa", "cose"},
|
||||
"fr": {"chose", "chose"},
|
||||
"de": {"thing", "things"},
|
||||
} {
|
||||
noun, plural := m.ResourceNouns(tag)
|
||||
Expect([2]string{noun, plural}).To(Equal(want), tag)
|
||||
}
|
||||
r.NounPlural = ""
|
||||
noun, plural := m.ResourceNouns("")
|
||||
Expect([2]string{noun, plural}).To(Equal([2]string{"thing", "thing"}))
|
||||
})
|
||||
It("refuses translated nouns the manifest does not name, empty ones and markup", func() {
|
||||
for name, mutate := range map[string]func(*Manifest){
|
||||
"no english noun": func(m *Manifest) { m.Capabilities.Resources.Noun = "" },
|
||||
"empty": func(m *Manifest) { m.I18n["it"] = Translation{Resources: &ResourceText{}} },
|
||||
"markup": func(m *Manifest) { m.I18n["it"] = Translation{Resources: &ResourceText{Noun: "<b>x</b>"}} },
|
||||
"plural alone": func(m *Manifest) { m.Capabilities.Resources.Noun, m.Capabilities.Resources.NounPlural = "", "x" },
|
||||
} {
|
||||
m := load()
|
||||
m.Capabilities.Resources.Noun = "thing"
|
||||
m.I18n = map[string]Translation{"it": {Resources: &ResourceText{Noun: "cosa"}}}
|
||||
mutate(&m)
|
||||
Expect(ValidateManifest(m, compatible())).NotTo(Succeed(), name)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -31,6 +31,16 @@ const maxAcceptLanguageBytes = 1024
|
||||
type Translation struct {
|
||||
// Kinds translates connector kinds, by kind ID.
|
||||
Kinds map[string]KindText `json:"kinds,omitempty"`
|
||||
// Resources translates the noun of the resources (feature level 2).
|
||||
Resources *ResourceText `json:"resources,omitempty"`
|
||||
}
|
||||
|
||||
// ResourceText is the translated noun of the resources of an add-on, singular
|
||||
// and plural. A field left out is shown in the next fallback (see
|
||||
// Manifest.ResourceNouns).
|
||||
type ResourceText struct {
|
||||
Noun string `json:"noun,omitempty"`
|
||||
NounPlural string `json:"nounPlural,omitempty"`
|
||||
}
|
||||
|
||||
// KindText is the translated text of one connector kind. A field left out is
|
||||
@@ -68,9 +78,21 @@ func validateI18n(m Manifest) error {
|
||||
return fmt.Errorf("i18n language %q: English is the manifest itself", tag)
|
||||
}
|
||||
seen[key] = true
|
||||
if len(t.Kinds) == 0 {
|
||||
if len(t.Kinds) == 0 && t.Resources == nil {
|
||||
return fmt.Errorf("i18n language %q translates nothing", tag)
|
||||
}
|
||||
if rt := t.Resources; rt != nil {
|
||||
rs := m.Capabilities.Resources
|
||||
if rs == nil || rs.Noun == "" || *rt == (ResourceText{}) {
|
||||
return fmt.Errorf("i18n %q translates resources that the manifest does not name", tag)
|
||||
}
|
||||
for _, w := range []string{rt.Noun, rt.NounPlural} {
|
||||
if w != "" && !plainText(w, 40) {
|
||||
return fmt.Errorf("i18n %q: invalid resource noun", tag)
|
||||
}
|
||||
total += utf8.RuneCountInString(w)
|
||||
}
|
||||
}
|
||||
for id, text := range t.Kinds {
|
||||
k, ok := kinds[id]
|
||||
if !ok {
|
||||
@@ -169,6 +191,37 @@ func (m Manifest) KindText(kindID, tag string) (title, description string) {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// ResourceNouns returns the singular and plural noun of the resources for a
|
||||
// language tag. A translation that gives only the singular is not mixed with
|
||||
// the English plural: the plural then equals its singular. Without a
|
||||
// matching translation the English plural is used, else the English noun.
|
||||
func (m Manifest) ResourceNouns(tag string) (noun, plural string) {
|
||||
r := m.Capabilities.Resources
|
||||
if r == nil {
|
||||
return "", ""
|
||||
}
|
||||
noun, plural = r.Noun, r.NounPlural
|
||||
if plural == "" {
|
||||
plural = noun
|
||||
}
|
||||
langs := make([]string, 0, len(m.I18n))
|
||||
for t, v := range m.I18n {
|
||||
if v.Resources != nil {
|
||||
langs = append(langs, t)
|
||||
}
|
||||
}
|
||||
if got, ok := MatchLanguage(tag, langs); ok {
|
||||
t := m.I18n[got].Resources
|
||||
if t.Noun != "" {
|
||||
noun, plural = t.Noun, t.Noun
|
||||
}
|
||||
if t.NounPlural != "" {
|
||||
plural = t.NounPlural
|
||||
}
|
||||
}
|
||||
return noun, plural
|
||||
}
|
||||
|
||||
// ValidLanguageTag reports whether s is a language tag a manifest may use: a
|
||||
// BCP 47 subset of a two or three letter language and up to three subtags.
|
||||
func ValidLanguageTag(s string) bool { return len(s) <= 35 && languageTag.MatchString(s) }
|
||||
|
||||
@@ -67,14 +67,69 @@ func (r VersionRange) isHostAPI() bool {
|
||||
return r.Min == HostAPIVersion && r.Max == HostAPIVersion
|
||||
}
|
||||
|
||||
// HostFeatureLevel is the feature level of this host. The level counts the
|
||||
// additive manifest features a host understands, apart from the integer
|
||||
// versions above, so a new optional field does not make every published
|
||||
// add-on incompatible. A manifest without a featureLevel needs level 1. A host
|
||||
// older than the first level-2 field reads the field as unknown and refuses
|
||||
// the manifest, as it refuses any unknown field.
|
||||
//
|
||||
// Level 2: capabilities.resources.agentScoped.
|
||||
const HostFeatureLevel uint64 = 2
|
||||
|
||||
// BaseFeatureLevel is the level of a manifest that sets no featureLevel.
|
||||
const BaseFeatureLevel uint64 = 1
|
||||
|
||||
// MaxFeatureLevel bounds the featureLevel of a manifest.
|
||||
const MaxFeatureLevel uint64 = 1000
|
||||
|
||||
// FeatureLevelRefusal is the refusal of an add-on that needs a newer feature
|
||||
// level than the host has. Its text is meant for people.
|
||||
type FeatureLevelRefusal struct{ Need, Host uint64 }
|
||||
|
||||
func (e *FeatureLevelRefusal) Error() string { return "needs a newer LocalAI" }
|
||||
|
||||
// EffectiveFeatureLevel is the level a host tuple stands for: an unset level
|
||||
// is the base level.
|
||||
func (h HostCompatibility) EffectiveFeatureLevel() uint64 {
|
||||
if h.FeatureLevel == 0 {
|
||||
return BaseFeatureLevel
|
||||
}
|
||||
return h.FeatureLevel
|
||||
}
|
||||
|
||||
// RequiredFeatureLevel is the level the manifest declares, 1 when it sets none.
|
||||
func (m Manifest) RequiredFeatureLevel() uint64 {
|
||||
if m.FeatureLevel == 0 {
|
||||
return BaseFeatureLevel
|
||||
}
|
||||
return m.FeatureLevel
|
||||
}
|
||||
|
||||
// usedFeatureLevel is the lowest level that holds every optional field the
|
||||
// manifest uses.
|
||||
func (m Manifest) usedFeatureLevel() uint64 {
|
||||
if r := m.Capabilities.Resources; r != nil && (r.AgentScoped || r.NounPlural != "") {
|
||||
return 2
|
||||
}
|
||||
for _, t := range m.I18n {
|
||||
if t.Resources != nil {
|
||||
return 2
|
||||
}
|
||||
}
|
||||
return BaseFeatureLevel
|
||||
}
|
||||
|
||||
type HostCompatibility struct {
|
||||
Host uint64 `json:"host"`
|
||||
API uint64 `json:"api"`
|
||||
Protocol uint64 `json:"protocol"`
|
||||
SDK uint64 `json:"sdk"`
|
||||
DataFormat uint64 `json:"dataFormat"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
// FeatureLevel is the feature level of the host; 0 means the base level.
|
||||
FeatureLevel uint64 `json:"featureLevel,omitempty"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
type Manifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
@@ -87,6 +142,12 @@ type Manifest struct {
|
||||
API VersionRange `json:"api"`
|
||||
Protocol VersionRange `json:"protocol"`
|
||||
DataFormat VersionRange `json:"dataFormat"`
|
||||
// FeatureLevel is the feature level the manifest needs from the host. It is
|
||||
// optional and defaults to 1. A manifest that uses a field of a higher
|
||||
// level must say so; a host with a lower level refuses it with a plain
|
||||
// reason. A host that predates the field refuses the manifest as it
|
||||
// refuses any unknown field.
|
||||
FeatureLevel uint64 `json:"featureLevel,omitempty"`
|
||||
// LaunchMode is absent or "always": the owner starts the addon while it is
|
||||
// enabled. "on-demand" is reserved and rejected until it is implemented.
|
||||
LaunchMode string `json:"launchMode,omitempty"`
|
||||
@@ -157,6 +218,16 @@ type Capabilities struct {
|
||||
type ResourceTools struct {
|
||||
// Noun is the singular word for one resource in the UI, such as database.
|
||||
Noun string `json:"noun,omitempty"`
|
||||
// NounPlural is the plural of Noun, such as databases. It is optional and
|
||||
// needs feature level 2; without it the UI uses the noun alone. A
|
||||
// translation of both words is under i18n.<language>.resources.
|
||||
NounPlural string `json:"nounPlural,omitempty"`
|
||||
// AgentScoped opts the add-on in to per-agent resource lists: a saved
|
||||
// agent then names the resources it may use and the mode of each, and the
|
||||
// host enforces the list on every tool call that takes a resource. It
|
||||
// needs feature level 2 and a List mapping, because the host builds the
|
||||
// choices from the list tool.
|
||||
AgentScoped bool `json:"agentScoped,omitempty"`
|
||||
// Create is a tool of Tools whose definition has the kind create.
|
||||
Create string `json:"create,omitempty"`
|
||||
// Rename is a tool of Tools whose definition has the kind schema and takes
|
||||
@@ -213,6 +284,12 @@ func validateResourceTools(c Capabilities) error {
|
||||
if r.Noun != "" && !plainText(r.Noun, 40) {
|
||||
return fmt.Errorf("invalid resource noun")
|
||||
}
|
||||
if r.NounPlural != "" && (r.Noun == "" || !plainText(r.NounPlural, 40)) {
|
||||
return fmt.Errorf("invalid resource plural noun")
|
||||
}
|
||||
if r.AgentScoped && r.List == nil {
|
||||
return fmt.Errorf("agentScoped resources need a list mapping")
|
||||
}
|
||||
for _, want := range []struct{ name, kind, label string }{
|
||||
{r.Create, ToolKindCreate, "create"}, {r.Rename, ToolKindSchema, "rename"}, {r.Inspect, ToolKindRead, "inspect"},
|
||||
} {
|
||||
@@ -363,7 +440,7 @@ func ParseManifest(data []byte) (Manifest, error) {
|
||||
m.Capabilities.ToolDefinitions[i].InputSchema = c
|
||||
}
|
||||
// Parsing validates intrinsic constraints without imposing this machine's platform.
|
||||
h := HostCompatibility{Host: m.Host.Min, API: m.API.Min, Protocol: m.Protocol.Min, DataFormat: m.DataFormat.Min, OS: m.OS, Arch: m.Arch}
|
||||
h := HostCompatibility{Host: m.Host.Min, API: m.API.Min, Protocol: m.Protocol.Min, DataFormat: m.DataFormat.Min, FeatureLevel: m.RequiredFeatureLevel(), OS: m.OS, Arch: m.Arch}
|
||||
if m.Capabilities.UI != nil {
|
||||
h.SDK = m.Capabilities.UI.SDK.Min
|
||||
}
|
||||
@@ -390,6 +467,15 @@ func ValidateManifest(m Manifest, h HostCompatibility) error {
|
||||
if !m.Host.accepts(h.Host) || !m.API.accepts(h.API) || !m.Protocol.accepts(h.Protocol) || !m.DataFormat.accepts(h.DataFormat) {
|
||||
return fmt.Errorf("incompatible host, API, protocol or data format")
|
||||
}
|
||||
if m.FeatureLevel > MaxFeatureLevel {
|
||||
return fmt.Errorf("invalid featureLevel")
|
||||
}
|
||||
if used := m.usedFeatureLevel(); m.RequiredFeatureLevel() < used {
|
||||
return fmt.Errorf("the manifest uses a field of feature level %d and must declare featureLevel %d", used, used)
|
||||
}
|
||||
if need := m.RequiredFeatureLevel(); need > h.EffectiveFeatureLevel() {
|
||||
return &FeatureLevelRefusal{Need: need, Host: h.EffectiveFeatureLevel()}
|
||||
}
|
||||
c := m.Capabilities
|
||||
if (!c.Service && len(c.Tools) == 0 && c.UI == nil && c.Connector == nil) || !uniqueIDs(c.Tools) {
|
||||
return fmt.Errorf("invalid capabilities")
|
||||
|
||||
@@ -16,7 +16,7 @@ func fixture() []byte {
|
||||
return b
|
||||
}
|
||||
func compatible() HostCompatibility {
|
||||
return HostCompatibility{Host: 1, API: 2, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"}
|
||||
return HostCompatibility{Host: 1, API: 2, Protocol: 1, SDK: 1, DataFormat: 1, FeatureLevel: HostFeatureLevel, OS: "linux", Arch: "amd64"}
|
||||
}
|
||||
|
||||
var _ = Describe("Manifest", func() {
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
// does not contain them is not installable and must not be published.
|
||||
func hostFor(platform string) contract.HostCompatibility {
|
||||
os_, arch, _ := strings.Cut(platform, "/")
|
||||
return contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: os_, Arch: arch}
|
||||
return contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: os_, Arch: arch}
|
||||
}
|
||||
|
||||
type fileEntry struct {
|
||||
|
||||
@@ -50,7 +50,13 @@ type catalogRelease struct {
|
||||
// SkillShadows repeats catalog.Release.SkillShadows: the built-in skills
|
||||
// the manifest's skill definitions replace.
|
||||
SkillShadows []string `json:"skillShadows,omitempty"`
|
||||
UI struct {
|
||||
// Requires repeats catalog.Release.Requires for the one field the
|
||||
// generator writes: the feature level of a package that needs more than
|
||||
// the base level.
|
||||
Requires struct {
|
||||
FeatureLevel uint64 `json:"featureLevel,omitempty"`
|
||||
} `json:"requires,omitzero"`
|
||||
UI struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Titles map[string]string `json:"titles,omitempty"`
|
||||
@@ -140,6 +146,19 @@ func translationFieldsYAML(ui *contract.UI, indent string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// requiresFieldsYAML renders the requires block of a library entry: the
|
||||
// feature level of a manifest that needs more than the base level. A manifest
|
||||
// at the base level has no block. The index reader of a host that predates
|
||||
// the field refuses the whole index when one entry has it (the reader is
|
||||
// strict), so the catalog command writes it only with
|
||||
// --feature-level-in-index.
|
||||
func requiresFieldsYAML(m contract.Manifest, indent string) string {
|
||||
if m.RequiredFeatureLevel() <= contract.BaseFeatureLevel {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%srequires:\n%s featureLevel: %d\n", indent, indent, m.RequiredFeatureLevel())
|
||||
}
|
||||
|
||||
// cmdCatalogTools adds the toolCount and tools fields to the entries of an
|
||||
// existing library index, reading each manifest from addons/<id>/manifest.json
|
||||
// of the repository. It touches no other field, so the digests and artifacts
|
||||
@@ -231,6 +250,7 @@ func cmdCatalog(args []string) error {
|
||||
out := fs.String("out", "", "output file")
|
||||
root := fs.String("root", ".", "with --from-manifest: repository root that the manifest path of the platform matrix is relative to")
|
||||
fromManifest := fs.Bool("from-manifest", false, "development channel only: render from the manifest and the enabled platforms, without a built package")
|
||||
featureLevelInIndex := fs.Bool("feature-level-in-index", false, "write requires.featureLevel for an add-on that needs more than level 1. A host that predates the field refuses the whole index, so leave it off while such hosts are in use")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -297,6 +317,9 @@ func cmdCatalog(args []string) error {
|
||||
// text lands inside it.
|
||||
text = strings.TrimRight(text, "\n") + "\n" + translationFieldsYAML(ui, " ") + toolFieldsYAML(manifest.Capabilities.Tools, "", contract.ShadowedTools(manifest.Capabilities.ToolDefinitions)...) +
|
||||
skillFieldsYAML(contract.ShadowedSkills(manifest.Capabilities.SkillDefinitions), "")
|
||||
if *featureLevelInIndex {
|
||||
text += requiresFieldsYAML(*manifest, "")
|
||||
}
|
||||
if mutable {
|
||||
text = strings.Replace(text, "digest: \"{{digest}}\"", "mutable: true", 1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package packaging
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("feature level in the library index", func() {
|
||||
render := func(id string, edit func(string) string, extra ...string) []catalogRelease {
|
||||
GinkgoHelper()
|
||||
p := newProjectFor(id, both)
|
||||
path := filepath.Join(p.root, "addons", id, "manifest.json")
|
||||
b, err := os.ReadFile(path)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(os.WriteFile(path, []byte(edit(string(b))), 0o644)).To(Succeed())
|
||||
out := filepath.Join(GinkgoT().TempDir(), "catalog.yaml")
|
||||
args := append([]string{"catalog", "--config", p.config, "--root", p.root, "--template", "../../../gallery/addons/" + id + ".yaml.tmpl", "--from-manifest", "--repository", newRegistry().repo("addons"), "--channel", "development", "--owner-repo", "example/localai", "--signer-ref", "refs/heads/master", "--out", out}, extra...)
|
||||
_, err = run(args...)
|
||||
Expect(err).NotTo(HaveOccurred(), id)
|
||||
raw, err := os.ReadFile(out)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
entries, err := readCatalogIndex(raw)
|
||||
Expect(err).NotTo(HaveOccurred(), id)
|
||||
Expect(entries).NotTo(BeEmpty())
|
||||
return entries
|
||||
}
|
||||
needsTwo := func(s string) string {
|
||||
return strings.Replace(s, `"schemaVersion": 1,`, `"schemaVersion": 1, "featureLevel": 2,`, 1)
|
||||
}
|
||||
It("writes nothing for a manifest at the base level, with or without the flag", func() {
|
||||
for _, extra := range [][]string{nil, {"--feature-level-in-index"}} {
|
||||
for _, e := range render("telegram", func(s string) string { return s }, extra...) {
|
||||
Expect(e.Requires.FeatureLevel).To(BeZero())
|
||||
}
|
||||
}
|
||||
})
|
||||
It("keeps the level out of the index by default, so a host that predates the field still reads it", func() {
|
||||
for _, e := range render("telegram", needsTwo) {
|
||||
Expect(e.Requires.FeatureLevel).To(BeZero())
|
||||
}
|
||||
})
|
||||
It("writes the level the manifest needs when asked to", func() {
|
||||
for _, e := range render("telegram", needsTwo, "--feature-level-in-index") {
|
||||
Expect(e.Requires.FeatureLevel).To(Equal(uint64(2)))
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -174,7 +174,7 @@ func checkManifest(t *testing.T, s Subject) (contract.Manifest, contract.Provide
|
||||
if m, err = contract.ParseManifest(s.Manifest); err != nil {
|
||||
t.Fatalf("manifest: %v", err)
|
||||
}
|
||||
host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, OS: m.OS, Arch: m.Arch}
|
||||
host := contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: contract.HostSDKVersion, DataFormat: 1, FeatureLevel: contract.HostFeatureLevel, OS: m.OS, Arch: m.Arch}
|
||||
if err := contract.ValidateManifest(m, host); err != nil {
|
||||
t.Fatalf("the host refuses the manifest: %v", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user