chore: merge master into distributed transport PR

Keep the newer SQLite dependency from master to resolve the conflict.

Assisted-by: Codex:gpt-6

scripts/build/gallery/main.go

@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: MIT
+// Package the official index and its repository-local base configurations.
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func main() {
+	if len(os.Args) != 4 {
+		fmt.Fprintln(os.Stderr, "usage: gallery REPOSITORY {gallery|backend} OUTPUT")
+		os.Exit(1)
+	}
+	if err := packageGallery(os.Args[1], os.Args[2], os.Args[3]); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func packageGallery(root, source, output string) error {
+	if source != "gallery" && source != "backend" {
+		return fmt.Errorf("unsupported gallery directory %q", source)
+	}
+	repository, err := os.OpenRoot(root)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = repository.Close() }()
+	body, err := repository.ReadFile(filepath.Join(source, "index.yaml"))
+	if err != nil {
+		return err
+	}
+	var doc yaml.Node
+	if err := yaml.Unmarshal(body, &doc); err != nil {
+		return err
+	}
+	// The build operator explicitly selects the output directory via the CLI.
+	if err := os.MkdirAll(output, 0700); err != nil { // #nosec G703 -- caller-selected output root
+		return err
+	}
+	destination, err := os.OpenRoot(output)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = destination.Close() }()
+	// Keep the tree relative to the repository root so repeated base configs
+	// share a layer, even when an index refers outside its own directory.
+	const prefix = "github:mudler/LocalAI/"
+	var walk func(*yaml.Node) error
+	walk = func(n *yaml.Node) error {
+		if n.Kind == yaml.MappingNode {
+			for i := 0; i < len(n.Content); i += 2 {
+				value := n.Content[i+1]
+				if n.Content[i].Value != "url" || value.Kind != yaml.ScalarNode || !strings.HasPrefix(value.Value, prefix) || !strings.HasSuffix(value.Value, "@master") {
+					continue
+				}
+				path := strings.TrimSuffix(strings.TrimPrefix(value.Value, prefix), "@master")
+				if !filepath.IsLocal(path) {
+					return fmt.Errorf("base config escapes repository: %q", path)
+				}
+				config, err := repository.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				if err := destination.MkdirAll(filepath.Dir(path), 0700); err != nil {
+					return err
+				}
+				if err := destination.WriteFile(path, config, 0600); err != nil {
+					return err
+				}
+				value.Value = filepath.ToSlash(path)
+			}
+		}
+		for _, child := range n.Content {
+			if err := walk(child); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	if err := walk(&doc); err != nil {
+		return err
+	}
+	body, err = yaml.Marshal(&doc)
+	if err != nil {
+		return err
+	}
+	return destination.WriteFile("index.yaml", body, 0600)
+}

scripts/build/gallery/main_test.go

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: MIT
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestGalleryPackage(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Gallery packaging") }
+
+var _ = Describe("Gallery packaging", func() {
+	It("packages both official indexes with every repository-local base available offline", func() {
+		for _, source := range []string{"gallery", "backend"} {
+			out := GinkgoT().TempDir()
+			Expect(packageGallery("../../..", source, out)).To(Succeed())
+			body, err := os.ReadFile(filepath.Join(out, "index.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+			var entries []map[string]any
+			Expect(yaml.Unmarshal(body, &entries)).To(Succeed())
+			Expect(entries).ToNot(BeEmpty())
+			for _, entry := range entries {
+				url, _ := entry["url"].(string)
+				Expect(url).ToNot(HavePrefix("github:mudler/LocalAI/"))
+				if strings.HasPrefix(url, "gallery/") {
+					Expect(filepath.Join(out, url)).To(BeAnExistingFile())
+				}
+			}
+		}
+	})
+	It("bundles local base configs and preserves external URLs and YAML aliases", func() {
+		root := GinkgoT().TempDir()
+		Expect(os.MkdirAll(filepath.Join(root, "gallery"), 0755)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(root, "gallery/base.yaml"), []byte("backend: llama-cpp\n"), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(root, "gallery/index.yaml"), []byte("- &base\n  name: first\n  url: github:mudler/LocalAI/gallery/base.yaml@master\n- <<: *base\n  name: second\n- name: external\n  url: https://example.com/config.yaml\n"), 0644)).To(Succeed())
+		out := filepath.Join(root, "out")
+		Expect(packageGallery(root, "gallery", out)).To(Succeed())
+		data, err := os.ReadFile(filepath.Join(out, "index.yaml"))
+		Expect(err).ToNot(HaveOccurred())
+		var entries []map[string]any
+		Expect(yaml.Unmarshal(data, &entries)).To(Succeed())
+		Expect(entries[0]["url"]).To(Equal("gallery/base.yaml"))
+		Expect(entries[1]["url"]).To(Equal("gallery/base.yaml"))
+		Expect(entries[2]["url"]).To(Equal("https://example.com/config.yaml"))
+		body, err := os.ReadFile(filepath.Join(out, "gallery/base.yaml"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(string(body)).To(Equal("backend: llama-cpp\n"))
+	})
+	It("fails if a referenced config is missing or escapes the repository", func() {
+		for _, ref := range []string{"missing.yaml", "../../outside.yaml"} {
+			root := GinkgoT().TempDir()
+			Expect(os.Mkdir(filepath.Join(root, "gallery"), 0755)).To(Succeed())
+			Expect(os.WriteFile(filepath.Join(root, "gallery/index.yaml"), []byte("- name: broken\n  url: github:mudler/LocalAI/gallery/"+ref+"@master\n"), 0644)).To(Succeed())
+			Expect(packageGallery(root, "gallery", filepath.Join(root, "out"))).ToNot(Succeed())
+		}
+	})
+	It("rejects symlink escapes when reading configs or writing the bundle", func() {
+		for _, location := range []string{"source", "output"} {
+			root, out, outside := GinkgoT().TempDir(), GinkgoT().TempDir(), GinkgoT().TempDir()
+			for _, dir := range []string{filepath.Join(root, "gallery"), filepath.Join(out, "gallery")} {
+				Expect(os.Mkdir(dir, 0700)).To(Succeed())
+			}
+			index := []byte("- name: test\n  url: github:mudler/LocalAI/gallery/base.yaml@master\n")
+			Expect(os.WriteFile(filepath.Join(root, "gallery/index.yaml"), index, 0600)).To(Succeed())
+			outsideFile := filepath.Join(outside, "base.yaml")
+			Expect(os.WriteFile(outsideFile, []byte("outside"), 0600)).To(Succeed())
+			link := filepath.Join(root, "gallery/base.yaml")
+			if location == "output" {
+				Expect(os.WriteFile(link, []byte("inside"), 0600)).To(Succeed())
+				link = filepath.Join(out, "gallery/base.yaml")
+			}
+			Expect(os.Symlink(outsideFile, link)).To(Succeed())
+			Expect(packageGallery(root, "gallery", out)).ToNot(Succeed(), location)
+			data, err := os.ReadFile(outsideFile)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(string(data)).To(Equal("outside"))
+		}
+	})
+})