diff --git a/pkg/addons/contract/version.go b/pkg/addons/contract/version.go new file mode 100644 index 0000000000..cf859c33ad --- /dev/null +++ b/pkg/addons/contract/version.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: MIT +package contract + +import "strings" + +// CompareVersions orders two add-on versions by semantic versioning: it +// returns -1, 0 or 1. A pre-release sorts below its release. Build metadata +// does not occur in an add-on version. +func CompareVersions(a, b string) int { + pa, pb := splitVersion(a), splitVersion(b) + for i := 0; i < 3; i++ { + if pa.num[i] != pb.num[i] { + if pa.num[i] < pb.num[i] { + return -1 + } + return 1 + } + } + switch { + case pa.pre == "" && pb.pre == "": + return 0 + case pa.pre == "": + return 1 + case pb.pre == "": + return -1 + } + as, bs := strings.Split(pa.pre, "."), strings.Split(pb.pre, ".") + for i := 0; i < len(as) && i < len(bs); i++ { + if as[i] == bs[i] { + continue + } + an, aok := atoiBounded(as[i]) + bn, bok := atoiBounded(bs[i]) + switch { + case aok && bok: + if an < bn { + return -1 + } + return 1 + case aok: + return -1 + case bok: + return 1 + case as[i] < bs[i]: + return -1 + default: + return 1 + } + } + switch { + case len(as) < len(bs): + return -1 + case len(as) > len(bs): + return 1 + } + return 0 +} + +type parsedVersion struct { + num [3]int + pre string +} + +func splitVersion(v string) parsedVersion { + core, pre, _ := strings.Cut(v, "-") + var s parsedVersion + for i, p := range strings.SplitN(core, ".", 3) { + n, _ := atoiBounded(p) + s.num[i] = n + } + s.pre = pre + return s +} + +func atoiBounded(s string) (int, bool) { + if s == "" { + return 0, false + } + n := 0 + for _, r := range s { + if r < '0' || r > '9' || n > 1<<40 { + return 0, false + } + n = n*10 + int(r-'0') + } + return n, true +} diff --git a/pkg/addons/contract/version_test.go b/pkg/addons/contract/version_test.go new file mode 100644 index 0000000000..0a3a0c83a8 --- /dev/null +++ b/pkg/addons/contract/version_test.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +package contract + +import "testing" + +func TestCompareVersions(t *testing.T) { + cases := []struct { + a, b string + want int + }{ + {"1.0.0", "1.0.0", 0}, + {"1.0.1", "1.0.0", 1}, + {"1.9.0", "1.10.0", -1}, + {"2.0.0", "1.99.99", 1}, + {"1.0.0-rc.1", "1.0.0", -1}, + {"1.0.0", "1.0.0-rc.1", 1}, + {"1.0.0-rc.2", "1.0.0-rc.10", -1}, + {"1.0.0-alpha", "1.0.0-alpha.1", -1}, + {"1.0.0-1", "1.0.0-a", -1}, + {"0.0.0-dev.abc", "0.0.1", -1}, + } + for _, c := range cases { + if got := CompareVersions(c.a, c.b); got != c.want { + t.Errorf("CompareVersions(%q, %q) = %d, want %d", c.a, c.b, got, c.want) + } + if got := CompareVersions(c.b, c.a); got != -c.want { + t.Errorf("CompareVersions(%q, %q) = %d, want %d", c.b, c.a, got, -c.want) + } + } +} diff --git a/pkg/addons/packaging/packaging_test.go b/pkg/addons/packaging/packaging_test.go index c52a50d3e6..60d56ae366 100644 --- a/pkg/addons/packaging/packaging_test.go +++ b/pkg/addons/packaging/packaging_test.go @@ -2,6 +2,7 @@ package packaging import ( + "archive/zip" "bytes" "context" "encoding/json" @@ -17,6 +18,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/mudler/LocalAI/pkg/addons/zipformat" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -76,6 +78,42 @@ var _ = Describe("package layout", func() { Expect(string(m)).To(ContainSubstring(`"arch": "amd64"`)) }) + It("writes the zip of a verified layout with a digest manifest and a sha256 file", func() { + p := newProject(both) + dir := p.mustBuild(amd, "1.2.3", "bin") + out := filepath.Join(GinkgoT().TempDir(), "dist") + res, err := run("zip", "--config", p.config, "--layout", dir, "--platform", amd, "--version", "1.2.3", "--out", out) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ContainSubstring("tables-1.2.3-linux-amd64.zip")) + zipPath := filepath.Join(out, "tables-1.2.3-linux-amd64.zip") + raw, err := os.ReadFile(zipPath) + Expect(err).NotTo(HaveOccurred()) + sum, err := os.ReadFile(zipPath + ".sha256") + Expect(err).NotTo(HaveOccurred()) + h, _ := sha(raw) + Expect(string(sum)).To(Equal(strings.TrimPrefix(h, "sha256:") + " tables-1.2.3-linux-amd64.zip\n")) + zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw))) + Expect(err).NotTo(HaveOccurred()) + names := map[string]os.FileMode{} + for _, f := range zr.File { + names[f.Name] = f.Mode().Perm() + } + Expect(names).To(HaveKey(zipformat.ManifestName)) + Expect(names).To(HaveKey("manifest.json")) + Expect(names["tables"]).To(Equal(os.FileMode(0o755))) + Expect(names["ui/tables.js"]).To(Equal(os.FileMode(0o644))) + // The same layout gives the same bytes. + out2 := filepath.Join(GinkgoT().TempDir(), "dist") + _, err = run("zip", "--config", p.config, "--layout", dir, "--platform", amd, "--version", "1.2.3", "--out", out2) + Expect(err).NotTo(HaveOccurred()) + again, err := os.ReadFile(filepath.Join(out2, "tables-1.2.3-linux-amd64.zip")) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(raw)) + // A layout of another version is refused. + _, err = run("zip", "--config", p.config, "--layout", dir, "--platform", amd, "--version", "9.9.9", "--out", out) + Expect(err).To(HaveOccurred()) + }) + It("refuses a platform that the matrix has not enabled unless it is a local check", func() { p := newProject(map[string][]string{amd: nil}) _, err := p.build(arm, "1.2.3", exe("x")) diff --git a/pkg/addons/packaging/registry.go b/pkg/addons/packaging/registry.go index 8a211b6a03..285beb7238 100644 --- a/pkg/addons/packaging/registry.go +++ b/pkg/addons/packaging/registry.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "github.com/mudler/LocalAI/pkg/addons/contract" "os" "os/exec" "regexp" @@ -281,81 +282,7 @@ func readAlias(ctx context.Context, repo name.Repository, addon, alias string) ( return st, nil } -func compareSemver(a, b string) int { - pa, pb := splitSemver(a), splitSemver(b) - for i := 0; i < 3; i++ { - if pa.num[i] != pb.num[i] { - if pa.num[i] < pb.num[i] { - return -1 - } - return 1 - } - } - switch { - case pa.pre == "" && pb.pre == "": - return 0 - case pa.pre == "": - return 1 - case pb.pre == "": - return -1 - } - as, bs := strings.Split(pa.pre, "."), strings.Split(pb.pre, ".") - for i := 0; i < len(as) && i < len(bs); i++ { - if as[i] == bs[i] { - continue - } - an, aok := atoi(as[i]) - bn, bok := atoi(bs[i]) - switch { - case aok && bok: - if an < bn { - return -1 - } - return 1 - case aok: - return -1 - case bok: - return 1 - case as[i] < bs[i]: - return -1 - default: - return 1 - } - } - return len(as) - len(bs) -} - -type semver struct { - num [3]int - pre string -} - -func splitSemver(v string) semver { - core, pre, _ := strings.Cut(v, "-") - var s semver - for i, p := range strings.SplitN(core, ".", 3) { - n, _ := atoi(p) - if i < 3 { - s.num[i] = n - } - } - s.pre = pre - return s -} - -func atoi(s string) (int, bool) { - if s == "" { - return 0, false - } - n := 0 - for _, r := range s { - if r < '0' || r > '9' || n > 1<<40 { - return 0, false - } - n = n*10 + int(r-'0') - } - return n, true -} +func compareSemver(a, b string) int { return contract.CompareVersions(a, b) } // errCommitUnreachable reports that git does not have one of the commits. // It is not the same as "not an ancestor": the history that would answer the diff --git a/pkg/addons/packaging/run.go b/pkg/addons/packaging/run.go index 0037d38653..62e814f37b 100644 --- a/pkg/addons/packaging/run.go +++ b/pkg/addons/packaging/run.go @@ -23,6 +23,7 @@ var commands = map[string]func(args []string) error{ "build": cmdBuild, "verify": cmdVerify, "extract": cmdExtract, + "zip": cmdZip, "native-check": cmdNativeCheck, "merge": cmdMerge, "publish": cmdPublish, diff --git a/pkg/addons/packaging/zip.go b/pkg/addons/packaging/zip.go new file mode 100644 index 0000000000..fae26a6716 --- /dev/null +++ b/pkg/addons/packaging/zip.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MIT +package packaging + +import ( + "bytes" + "crypto/sha256" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/pkg/addons/zipformat" +) + +// cmdZip writes the zip of a verified platform layout, and its sha256 file. +// The files are those of the OCI package, so the zip is not a second build +// path: whatever the layout passed in verify is what the zip holds. +func cmdZip(args []string) error { + fs := flag.NewFlagSet("zip", flag.ContinueOnError) + dir := fs.String("layout", "", "platform OCI layout") + platform := fs.String("platform", "", "expected os/arch") + version := fs.String("version", "", "expected version") + config := fs.String("config", "addons/tables/package/platforms.yaml", "platform matrix file") + outDir := fs.String("out", "", "directory that receives ---.zip and its .sha256 file") + if err := fs.Parse(args); err != nil { + return err + } + if *outDir == "" || *dir == "" { + return errors.New("--layout and --out are required") + } + cfg, err := loadPlatforms(*config) + if err != nil { + return err + } + v, err := verifyLeg(*dir, *platform, *version, cfg) + if err != nil { + return err + } + entries := make([]zipformat.Entry, 0, len(v.Files)) + for _, f := range v.Files { + entries = append(entries, zipformat.Entry{Name: f.name, Data: f.data, Exec: f.exec}) + } + var buf bytes.Buffer + m := v.Manifest + if err := zipformat.Write(&buf, m.ID, m.Version, m.OS, m.Arch, m.Executable, entries); err != nil { + return err + } + if err := os.MkdirAll(*outDir, 0o750); err != nil { + return err + } + name := zipformat.FileName(m.ID, m.Version, m.OS, m.Arch) + zipPath := filepath.Join(*outDir, name) + if err := os.WriteFile(zipPath, buf.Bytes(), 0o644); err != nil { // #nosec G306 -- a distributable file + return err + } + sum := sha256.Sum256(buf.Bytes()) + line := fmt.Sprintf("%x %s\n", sum, name) + if err := os.WriteFile(zipPath+".sha256", []byte(line), 0o644); err != nil { // #nosec G306 -- a distributable file + return err + } + _, _ = fmt.Fprintf(stdout, "zip=%s\nsha256=%x\nsize=%d\n", zipPath, sum, buf.Len()) + return nil +} diff --git a/pkg/addons/zipformat/zipformat.go b/pkg/addons/zipformat/zipformat.go new file mode 100644 index 0000000000..13181ecac8 --- /dev/null +++ b/pkg/addons/zipformat/zipformat.go @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT + +// Package zipformat defines the zip file that carries one add-on package for +// one platform: the files of the OCI package plus a digest manifest. The build +// tool writes it and the owner's installer reads it, so both share these +// names and rules. +package zipformat + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "regexp" + "sort" + "time" +) + +// ManifestName is the digest manifest at the root of the zip. It is not a +// file of the package and the installer does not extract it. +const ManifestName = "zip-manifest.json" + +// Schema is the version of the digest manifest. +const Schema = 1 + +// SourceID, Verification and OriginLabel are what the receipt of a package +// installed from a local file records. Nothing signs such a package. +const ( + SourceID = "local-file" + Verification = "sha256" + OriginLabel = "local file, unsigned" +) + +const ( + zipModeFile fs.FileMode = 0o644 + zipModeExecute fs.FileMode = 0o755 +) + +// File is one file of the package. +type File struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +// Manifest lists the identity of the package and every file in it. +type Manifest struct { + Schema int `json:"schema"` + ID string `json:"id"` + Version string `json:"version"` + OS string `json:"os"` + Arch string `json:"arch"` + Executable string `json:"executable"` + Files []File `json:"files"` +} + +// Entry is a file to write. +type Entry struct { + Name string + Data []byte + Exec bool +} + +var shaRE = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// FileName is the name of the zip of a package: ---.zip. +func FileName(id, version, os, arch string) string { + return fmt.Sprintf("%s-%s-%s-%s.zip", id, version, os, arch) +} + +// ParseManifest decodes the digest manifest strictly. +func ParseManifest(b []byte) (Manifest, error) { + var m Manifest + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + if err := d.Decode(&m); err != nil { + return Manifest{}, err + } + if _, err := d.Token(); err != io.EOF { + return Manifest{}, errors.New("trailing data after the digest manifest") + } + if m.Schema != Schema || m.ID == "" || m.Version == "" || m.OS == "" || m.Arch == "" || m.Executable == "" || len(m.Files) == 0 { + return Manifest{}, errors.New("incomplete digest manifest") + } + seen := map[string]bool{} + for _, f := range m.Files { + if f.Path == "" || !shaRE.MatchString(f.SHA256) || f.Size < 0 || seen[f.Path] { + return Manifest{}, errors.New("invalid file entry in the digest manifest") + } + seen[f.Path] = true + } + return m, nil +} + +// Write writes the zip: the digest manifest first, then the files in sorted +// order with fixed metadata, so the same package gives the same bytes. +func Write(w io.Writer, id, version, os, arch, executable string, files []Entry) error { + sorted := append([]Entry(nil), files...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + m := Manifest{Schema: Schema, ID: id, Version: version, OS: os, Arch: arch, Executable: executable} + for _, f := range sorted { + sum := sha256.Sum256(f.Data) + m.Files = append(m.Files, File{Path: f.Name, SHA256: hex.EncodeToString(sum[:]), Size: int64(len(f.Data))}) + } + raw, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + zw := zip.NewWriter(w) + put := func(name string, data []byte, mode fs.FileMode) error { + h := &zip.FileHeader{Name: name, Method: zip.Deflate, Modified: time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC)} + h.SetMode(mode) + fw, err := zw.CreateHeader(h) + if err != nil { + return err + } + _, err = fw.Write(data) + return err + } + if err := put(ManifestName, append(raw, '\n'), zipModeFile); err != nil { + return err + } + for _, f := range sorted { + mode := zipModeFile + if f.Exec { + mode = zipModeExecute + } + if err := put(f.Name, f.Data, mode); err != nil { + return err + } + } + return zw.Close() +}