feat(addons): write one add-on package as a zip

Add the zip format of a package (a digest manifest plus the files of the
OCI package for one platform) and an addonpkg zip subcommand that writes
it from a verified layout, with a sha256 file. The zip comes from the
same layout the registry package comes from, so there is one build path.

Version comparison moves to contract.CompareVersions so the installer
can use it for upgrades and downgrades.

Assisted-by: Claude:claude-sonnet-5-5 go gofmt

pkg/addons/contract/version.go

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: MIT
+package contract
+
+import "strings"
+
+// CompareVersions orders two add-on versions by semantic versioning: it
+// returns -1, 0 or 1. A pre-release sorts below its release. Build metadata
+// does not occur in an add-on version.
+func CompareVersions(a, b string) int {
+	pa, pb := splitVersion(a), splitVersion(b)
+	for i := 0; i < 3; i++ {
+		if pa.num[i] != pb.num[i] {
+			if pa.num[i] < pb.num[i] {
+				return -1
+			}
+			return 1
+		}
+	}
+	switch {
+	case pa.pre == "" && pb.pre == "":
+		return 0
+	case pa.pre == "":
+		return 1
+	case pb.pre == "":
+		return -1
+	}
+	as, bs := strings.Split(pa.pre, "."), strings.Split(pb.pre, ".")
+	for i := 0; i < len(as) && i < len(bs); i++ {
+		if as[i] == bs[i] {
+			continue
+		}
+		an, aok := atoiBounded(as[i])
+		bn, bok := atoiBounded(bs[i])
+		switch {
+		case aok && bok:
+			if an < bn {
+				return -1
+			}
+			return 1
+		case aok:
+			return -1
+		case bok:
+			return 1
+		case as[i] < bs[i]:
+			return -1
+		default:
+			return 1
+		}
+	}
+	switch {
+	case len(as) < len(bs):
+		return -1
+	case len(as) > len(bs):
+		return 1
+	}
+	return 0
+}
+
+type parsedVersion struct {
+	num [3]int
+	pre string
+}
+
+func splitVersion(v string) parsedVersion {
+	core, pre, _ := strings.Cut(v, "-")
+	var s parsedVersion
+	for i, p := range strings.SplitN(core, ".", 3) {
+		n, _ := atoiBounded(p)
+		s.num[i] = n
+	}
+	s.pre = pre
+	return s
+}
+
+func atoiBounded(s string) (int, bool) {
+	if s == "" {
+		return 0, false
+	}
+	n := 0
+	for _, r := range s {
+		if r < '0' || r > '9' || n > 1<<40 {
+			return 0, false
+		}
+		n = n*10 + int(r-'0')
+	}
+	return n, true
+}

pkg/addons/contract/version_test.go

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: MIT
+package contract
+
+import "testing"
+
+func TestCompareVersions(t *testing.T) {
+	cases := []struct {
+		a, b string
+		want int
+	}{
+		{"1.0.0", "1.0.0", 0},
+		{"1.0.1", "1.0.0", 1},
+		{"1.9.0", "1.10.0", -1},
+		{"2.0.0", "1.99.99", 1},
+		{"1.0.0-rc.1", "1.0.0", -1},
+		{"1.0.0", "1.0.0-rc.1", 1},
+		{"1.0.0-rc.2", "1.0.0-rc.10", -1},
+		{"1.0.0-alpha", "1.0.0-alpha.1", -1},
+		{"1.0.0-1", "1.0.0-a", -1},
+		{"0.0.0-dev.abc", "0.0.1", -1},
+	}
+	for _, c := range cases {
+		if got := CompareVersions(c.a, c.b); got != c.want {
+			t.Errorf("CompareVersions(%q, %q) = %d, want %d", c.a, c.b, got, c.want)
+		}
+		if got := CompareVersions(c.b, c.a); got != -c.want {
+			t.Errorf("CompareVersions(%q, %q) = %d, want %d", c.b, c.a, got, -c.want)
+		}
+	}
+}

pkg/addons/packaging/packaging_test.go

@@ -2,6 +2,7 @@
 package packaging
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -17,6 +18,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/mudler/LocalAI/pkg/addons/zipformat"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -76,6 +78,42 @@ var _ = Describe("package layout", func() {
 		Expect(string(m)).To(ContainSubstring(`"arch": "amd64"`))
 	})
 
+	It("writes the zip of a verified layout with a digest manifest and a sha256 file", func() {
+		p := newProject(both)
+		dir := p.mustBuild(amd, "1.2.3", "bin")
+		out := filepath.Join(GinkgoT().TempDir(), "dist")
+		res, err := run("zip", "--config", p.config, "--layout", dir, "--platform", amd, "--version", "1.2.3", "--out", out)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res).To(ContainSubstring("tables-1.2.3-linux-amd64.zip"))
+		zipPath := filepath.Join(out, "tables-1.2.3-linux-amd64.zip")
+		raw, err := os.ReadFile(zipPath)
+		Expect(err).NotTo(HaveOccurred())
+		sum, err := os.ReadFile(zipPath + ".sha256")
+		Expect(err).NotTo(HaveOccurred())
+		h, _ := sha(raw)
+		Expect(string(sum)).To(Equal(strings.TrimPrefix(h, "sha256:") + "  tables-1.2.3-linux-amd64.zip\n"))
+		zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
+		Expect(err).NotTo(HaveOccurred())
+		names := map[string]os.FileMode{}
+		for _, f := range zr.File {
+			names[f.Name] = f.Mode().Perm()
+		}
+		Expect(names).To(HaveKey(zipformat.ManifestName))
+		Expect(names).To(HaveKey("manifest.json"))
+		Expect(names["tables"]).To(Equal(os.FileMode(0o755)))
+		Expect(names["ui/tables.js"]).To(Equal(os.FileMode(0o644)))
+		// The same layout gives the same bytes.
+		out2 := filepath.Join(GinkgoT().TempDir(), "dist")
+		_, err = run("zip", "--config", p.config, "--layout", dir, "--platform", amd, "--version", "1.2.3", "--out", out2)
+		Expect(err).NotTo(HaveOccurred())
+		again, err := os.ReadFile(filepath.Join(out2, "tables-1.2.3-linux-amd64.zip"))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(again).To(Equal(raw))
+		// A layout of another version is refused.
+		_, err = run("zip", "--config", p.config, "--layout", dir, "--platform", amd, "--version", "9.9.9", "--out", out)
+		Expect(err).To(HaveOccurred())
+	})
+
 	It("refuses a platform that the matrix has not enabled unless it is a local check", func() {
 		p := newProject(map[string][]string{amd: nil})
 		_, err := p.build(arm, "1.2.3", exe("x"))

pkg/addons/packaging/registry.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"github.com/mudler/LocalAI/pkg/addons/contract"
 	"os"
 	"os/exec"
 	"regexp"
@@ -281,81 +282,7 @@ func readAlias(ctx context.Context, repo name.Repository, addon, alias string) (
 	return st, nil
 }
 
-func compareSemver(a, b string) int {
-	pa, pb := splitSemver(a), splitSemver(b)
-	for i := 0; i < 3; i++ {
-		if pa.num[i] != pb.num[i] {
-			if pa.num[i] < pb.num[i] {
-				return -1
-			}
-			return 1
-		}
-	}
-	switch {
-	case pa.pre == "" && pb.pre == "":
-		return 0
-	case pa.pre == "":
-		return 1
-	case pb.pre == "":
-		return -1
-	}
-	as, bs := strings.Split(pa.pre, "."), strings.Split(pb.pre, ".")
-	for i := 0; i < len(as) && i < len(bs); i++ {
-		if as[i] == bs[i] {
-			continue
-		}
-		an, aok := atoi(as[i])
-		bn, bok := atoi(bs[i])
-		switch {
-		case aok && bok:
-			if an < bn {
-				return -1
-			}
-			return 1
-		case aok:
-			return -1
-		case bok:
-			return 1
-		case as[i] < bs[i]:
-			return -1
-		default:
-			return 1
-		}
-	}
-	return len(as) - len(bs)
-}
-
-type semver struct {
-	num [3]int
-	pre string
-}
-
-func splitSemver(v string) semver {
-	core, pre, _ := strings.Cut(v, "-")
-	var s semver
-	for i, p := range strings.SplitN(core, ".", 3) {
-		n, _ := atoi(p)
-		if i < 3 {
-			s.num[i] = n
-		}
-	}
-	s.pre = pre
-	return s
-}
-
-func atoi(s string) (int, bool) {
-	if s == "" {
-		return 0, false
-	}
-	n := 0
-	for _, r := range s {
-		if r < '0' || r > '9' || n > 1<<40 {
-			return 0, false
-		}
-		n = n*10 + int(r-'0')
-	}
-	return n, true
-}
+func compareSemver(a, b string) int { return contract.CompareVersions(a, b) }
 
 // errCommitUnreachable reports that git does not have one of the commits.
 // It is not the same as "not an ancestor": the history that would answer the

pkg/addons/packaging/run.go

@@ -23,6 +23,7 @@ var commands = map[string]func(args []string) error{
 	"build":         cmdBuild,
 	"verify":        cmdVerify,
 	"extract":       cmdExtract,
+	"zip":           cmdZip,
 	"native-check":  cmdNativeCheck,
 	"merge":         cmdMerge,
 	"publish":       cmdPublish,

pkg/addons/packaging/zip.go

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: MIT
+package packaging
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/pkg/addons/zipformat"
+)
+
+// cmdZip writes the zip of a verified platform layout, and its sha256 file.
+// The files are those of the OCI package, so the zip is not a second build
+// path: whatever the layout passed in verify is what the zip holds.
+func cmdZip(args []string) error {
+	fs := flag.NewFlagSet("zip", flag.ContinueOnError)
+	dir := fs.String("layout", "", "platform OCI layout")
+	platform := fs.String("platform", "", "expected os/arch")
+	version := fs.String("version", "", "expected version")
+	config := fs.String("config", "addons/tables/package/platforms.yaml", "platform matrix file")
+	outDir := fs.String("out", "", "directory that receives <id>-<version>-<os>-<arch>.zip and its .sha256 file")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *outDir == "" || *dir == "" {
+		return errors.New("--layout and --out are required")
+	}
+	cfg, err := loadPlatforms(*config)
+	if err != nil {
+		return err
+	}
+	v, err := verifyLeg(*dir, *platform, *version, cfg)
+	if err != nil {
+		return err
+	}
+	entries := make([]zipformat.Entry, 0, len(v.Files))
+	for _, f := range v.Files {
+		entries = append(entries, zipformat.Entry{Name: f.name, Data: f.data, Exec: f.exec})
+	}
+	var buf bytes.Buffer
+	m := v.Manifest
+	if err := zipformat.Write(&buf, m.ID, m.Version, m.OS, m.Arch, m.Executable, entries); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(*outDir, 0o750); err != nil {
+		return err
+	}
+	name := zipformat.FileName(m.ID, m.Version, m.OS, m.Arch)
+	zipPath := filepath.Join(*outDir, name)
+	if err := os.WriteFile(zipPath, buf.Bytes(), 0o644); err != nil { // #nosec G306 -- a distributable file
+		return err
+	}
+	sum := sha256.Sum256(buf.Bytes())
+	line := fmt.Sprintf("%x  %s\n", sum, name)
+	if err := os.WriteFile(zipPath+".sha256", []byte(line), 0o644); err != nil { // #nosec G306 -- a distributable file
+		return err
+	}
+	_, _ = fmt.Fprintf(stdout, "zip=%s\nsha256=%x\nsize=%d\n", zipPath, sum, buf.Len())
+	return nil
+}

pkg/addons/zipformat/zipformat.go

@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: MIT
+
+// Package zipformat defines the zip file that carries one add-on package for
+// one platform: the files of the OCI package plus a digest manifest. The build
+// tool writes it and the owner's installer reads it, so both share these
+// names and rules.
+package zipformat
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"regexp"
+	"sort"
+	"time"
+)
+
+// ManifestName is the digest manifest at the root of the zip. It is not a
+// file of the package and the installer does not extract it.
+const ManifestName = "zip-manifest.json"
+
+// Schema is the version of the digest manifest.
+const Schema = 1
+
+// SourceID, Verification and OriginLabel are what the receipt of a package
+// installed from a local file records. Nothing signs such a package.
+const (
+	SourceID     = "local-file"
+	Verification = "sha256"
+	OriginLabel  = "local file, unsigned"
+)
+
+const (
+	zipModeFile    fs.FileMode = 0o644
+	zipModeExecute fs.FileMode = 0o755
+)
+
+// File is one file of the package.
+type File struct {
+	Path   string `json:"path"`
+	SHA256 string `json:"sha256"`
+	Size   int64  `json:"size"`
+}
+
+// Manifest lists the identity of the package and every file in it.
+type Manifest struct {
+	Schema     int    `json:"schema"`
+	ID         string `json:"id"`
+	Version    string `json:"version"`
+	OS         string `json:"os"`
+	Arch       string `json:"arch"`
+	Executable string `json:"executable"`
+	Files      []File `json:"files"`
+}
+
+// Entry is a file to write.
+type Entry struct {
+	Name string
+	Data []byte
+	Exec bool
+}
+
+var shaRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// FileName is the name of the zip of a package: <id>-<version>-<os>-<arch>.zip.
+func FileName(id, version, os, arch string) string {
+	return fmt.Sprintf("%s-%s-%s-%s.zip", id, version, os, arch)
+}
+
+// ParseManifest decodes the digest manifest strictly.
+func ParseManifest(b []byte) (Manifest, error) {
+	var m Manifest
+	d := json.NewDecoder(bytes.NewReader(b))
+	d.DisallowUnknownFields()
+	if err := d.Decode(&m); err != nil {
+		return Manifest{}, err
+	}
+	if _, err := d.Token(); err != io.EOF {
+		return Manifest{}, errors.New("trailing data after the digest manifest")
+	}
+	if m.Schema != Schema || m.ID == "" || m.Version == "" || m.OS == "" || m.Arch == "" || m.Executable == "" || len(m.Files) == 0 {
+		return Manifest{}, errors.New("incomplete digest manifest")
+	}
+	seen := map[string]bool{}
+	for _, f := range m.Files {
+		if f.Path == "" || !shaRE.MatchString(f.SHA256) || f.Size < 0 || seen[f.Path] {
+			return Manifest{}, errors.New("invalid file entry in the digest manifest")
+		}
+		seen[f.Path] = true
+	}
+	return m, nil
+}
+
+// Write writes the zip: the digest manifest first, then the files in sorted
+// order with fixed metadata, so the same package gives the same bytes.
+func Write(w io.Writer, id, version, os, arch, executable string, files []Entry) error {
+	sorted := append([]Entry(nil), files...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
+	m := Manifest{Schema: Schema, ID: id, Version: version, OS: os, Arch: arch, Executable: executable}
+	for _, f := range sorted {
+		sum := sha256.Sum256(f.Data)
+		m.Files = append(m.Files, File{Path: f.Name, SHA256: hex.EncodeToString(sum[:]), Size: int64(len(f.Data))})
+	}
+	raw, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return err
+	}
+	zw := zip.NewWriter(w)
+	put := func(name string, data []byte, mode fs.FileMode) error {
+		h := &zip.FileHeader{Name: name, Method: zip.Deflate, Modified: time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC)}
+		h.SetMode(mode)
+		fw, err := zw.CreateHeader(h)
+		if err != nil {
+			return err
+		}
+		_, err = fw.Write(data)
+		return err
+	}
+	if err := put(ManifestName, append(raw, '\n'), zipModeFile); err != nil {
+		return err
+	}
+	for _, f := range sorted {
+		mode := zipModeFile
+		if f.Exec {
+			mode = zipModeExecute
+		}
+		if err := put(f.Name, f.Data, mode); err != nil {
+			return err
+		}
+	}
+	return zw.Close()
+}