ci(gallery): have the index bots write the model library (#12849)

* ci(gallery): have the checksum bot write the model library

The checksum bot edited gallery/index.yaml, which is a generated copy
of gallery/models/index.yaml. The model library check fails any PR
where the copy differs from the generated one, and an edit made to the
copy never reaches the source.

Point the checker at the library, read the files under .model.files as
the library keeps them, and run modelcatalog after the checker so the
PR carries both files.

Assisted-by: Claude:claude-sonnet-5-5 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* ci(gallery): have the other index bots write the model library

The gallery agent and the variant proposer still read and wrote the
generated gallery copy, so their pull requests failed the model
library check and their changes never reached the source.

The gallery agent now appends library entries to
gallery/models/index.yaml, converting its entries with the gallery
package and skipping ids the library already has. Its duplicate check
reads the same file.

The variant proposer decodes the library shape (the body under
model:) and splices variants into the model body through a library
layout in galleryedit. The anchor refusal goes: the library has no
anchors. Its tests use library entries.

Both workflows run modelcatalog after the bot and before the pull
request step, and use the Go version from go.mod.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

.github/checksum_checker.sh

@@ -2,7 +2,9 @@
 # This scripts needs yq and huggingface_hub to be installed
 # to install hugingface_hub run pip install huggingface_hub
 
-# Path to the input YAML file
+# Path to the input YAML file: the model library, gallery/models/index.yaml.
+# Its entries keep the files under .model.files. The caller regenerates the
+# gallery copy, gallery/index.yaml, after this script ran.
 input_yaml=$1
 
 # Function to download file and check checksum using Python
@@ -34,11 +36,11 @@ function check_and_update_checksum() {
     
     if [[ $result -eq 2 ]]; then
         echo "File not found, deleting entry for $file_name..."
-        # yq eval -i "del(.[$idx].files[] | select(.filename == \"$file_name\"))" "$input_yaml"
+        # yq eval -i "del(.[$idx].model.files[] | select(.filename == \"$file_name\"))" "$input_yaml"
     elif [[ "$old_checksum" != "$new_checksum" ]]; then
         echo "Checksum mismatch for $file_name. Updating..."
-        yq eval -i "del(.[$idx].files[] | select(.filename == \"$file_name\").sha256)" "$input_yaml"
-        yq eval -i "(.[$idx].files[] | select(.filename == \"$file_name\")).sha256 = \"$new_checksum\"" "$input_yaml"
+        yq eval -i "del(.[$idx].model.files[] | select(.filename == \"$file_name\").sha256)" "$input_yaml"
+        yq eval -i "(.[$idx].model.files[] | select(.filename == \"$file_name\")).sha256 = \"$new_checksum\"" "$input_yaml"
     elif [[ $result -ne 0 ]]; then
         echo "Error downloading file $file_name. Skipping..."
     else
@@ -51,12 +53,12 @@ len=$(yq eval '. | length' "$input_yaml")
 for ((i=0; i<$len; i++))
 do
     name=$(yq eval ".[$i].name" "$input_yaml")
-    files_len=$(yq eval ".[$i].files | length" "$input_yaml")
+    files_len=$(yq eval ".[$i].model.files | length" "$input_yaml")
     for ((j=0; j<$files_len; j++))
     do
-        filename=$(yq eval ".[$i].files[$j].filename" "$input_yaml")
-        uri=$(yq eval ".[$i].files[$j].uri" "$input_yaml")
-        checksum=$(yq eval ".[$i].files[$j].sha256" "$input_yaml")
+        filename=$(yq eval ".[$i].model.files[$j].filename" "$input_yaml")
+        uri=$(yq eval ".[$i].model.files[$j].uri" "$input_yaml")
+        checksum=$(yq eval ".[$i].model.files[$j].sha256" "$input_yaml")
         echo "Checking model $name, file $filename. URI = $uri, Checksum = $checksum"
         check_and_update_checksum "$name" "$filename" "$uri" "$checksum" "$i"
     done

.github/ci/galleryedit/edit.go

@@ -25,6 +25,35 @@ var (
 	unsafeInName = regexp.MustCompile(`[:#{}\[\],&*?|>'"%@` + "`" + `]|^\s|\s$`)
 )
 
+// layout says where the variants key and its items sit in an entry. The
+// gallery keeps them at the top of the entry, the model library keeps them in
+// the model body.
+type layout struct {
+	keyVariants *regexp.Regexp
+	variantItem *regexp.Regexp
+	keyLine     string
+	itemPrefix  string
+	// afterName puts a new variants key right after the name line. Without
+	// it a new key goes after the last line of the entry, where the library
+	// keeps the end of the model body.
+	afterName bool
+}
+
+var galleryLayout = layout{
+	keyVariants: keyVariants,
+	variantItem: variantItem,
+	keyLine:     "  variants:",
+	itemPrefix:  "    - model: ",
+	afterName:   true,
+}
+
+var libraryLayout = layout{
+	keyVariants: regexp.MustCompile(`^    variants:\s*(.*)$`),
+	variantItem: regexp.MustCompile(`^      - `),
+	keyLine:     "    variants:",
+	itemPrefix:  "      - model: ",
+}
+
 // Entry is the positional view of one gallery entry: what it is called and
 // which lines it occupies. Nothing about what the entry MEANS belongs here, so
 // each caller keeps its own semantic decode and only hands over the coordinates.
@@ -55,8 +84,19 @@ func Scan(text string) (lines []string, starts []int) {
 	return lines, starts
 }
 
-// Apply splices every insert into the index lines and returns the new text.
+// Apply splices every insert into the gallery index lines and returns the new
+// text.
 func Apply(lines []string, inserts []Insert) ([]string, error) {
+	return apply(lines, inserts, galleryLayout)
+}
+
+// ApplyLibrary does the same for the model library, gallery/models/index.yaml,
+// where an entry keeps its variants under the model: body.
+func ApplyLibrary(lines []string, inserts []Insert) ([]string, error) {
+	return apply(lines, inserts, libraryLayout)
+}
+
+func apply(lines []string, inserts []Insert, l layout) ([]string, error) {
 	type edit struct {
 		at     int
 		remove int
@@ -71,16 +111,16 @@ func Apply(lines []string, inserts []Insert) ([]string, error) {
 
 		items := make([]string, 0, len(in.Variants))
 		for _, v := range in.Variants {
-			items = append(items, "    - model: "+QuoteName(v))
+			items = append(items, l.itemPrefix+QuoteName(v))
 		}
 
-		at, remove, err := insertionPoint(lines, in.Entry)
+		at, remove, err := insertionPoint(lines, in.Entry, l)
 		if err != nil {
 			return nil, err
 		}
 		block := items
-		if remove > 0 || !hasVariantsKey(lines, in.Entry) {
-			block = append([]string{"  variants:"}, items...)
+		if remove > 0 || !hasVariantsKey(lines, in.Entry, l) {
+			block = append([]string{l.keyLine}, items...)
 		}
 		edits = append(edits, edit{at: at, remove: remove, insert: block})
 	}
@@ -97,9 +137,9 @@ func Apply(lines []string, inserts []Insert) ([]string, error) {
 	return out, nil
 }
 
-func hasVariantsKey(lines []string, e Entry) bool {
+func hasVariantsKey(lines []string, e Entry, l layout) bool {
 	for i := e.StartLine; i < e.EndLine; i++ {
-		if keyVariants.MatchString(lines[i]) {
+		if l.keyVariants.MatchString(lines[i]) {
 			return true
 		}
 	}
@@ -112,9 +152,9 @@ func hasVariantsKey(lines []string, e Entry) bool {
 // An entry with no variants key gets one right after its name, which is where
 // the hand-written families put it. An entry with an empty "variants: []" has
 // that line replaced by a block. An entry with a block gets its items appended.
-func insertionPoint(lines []string, e Entry) (at int, remove int, err error) {
+func insertionPoint(lines []string, e Entry, l layout) (at int, remove int, err error) {
 	for i := e.StartLine; i < e.EndLine; i++ {
-		m := keyVariants.FindStringSubmatch(lines[i])
+		m := l.keyVariants.FindStringSubmatch(lines[i])
 		if m == nil {
 			continue
 		}
@@ -125,12 +165,19 @@ func insertionPoint(lines []string, e Entry) (at int, remove int, err error) {
 			return 0, 0, fmt.Errorf("entry %q writes its variants inline (%q); this job only edits block lists", e.Name, strings.TrimSpace(m[1]))
 		}
 		last := i
-		for j := i + 1; j < e.EndLine && variantItem.MatchString(lines[j]); j++ {
+		for j := i + 1; j < e.EndLine && l.variantItem.MatchString(lines[j]); j++ {
 			last = j
 		}
 		return last + 1, 0, nil
 	}
 
+	if !l.afterName {
+		last := e.EndLine - 1
+		for last > e.StartLine && strings.TrimSpace(lines[last]) == "" {
+			last--
+		}
+		return last + 1, 0, nil
+	}
 	if inlineName.MatchString(lines[e.StartLine]) {
 		return e.StartLine + 1, 0, nil
 	}

.github/ci/variantproposals/edit.go

@@ -7,7 +7,7 @@ import (
 	"github.com/mudler/LocalAI/.github/ci/galleryedit"
 )
 
-// ApplyFamilies writes the proposed variant lists into the index text.
+// ApplyFamilies writes the proposed variant lists into the library text.
 //
 // The line editing itself lives in galleryedit, shared with the apexentries
 // generator. Both jobs add variants to entries the gallery already ships, and a
@@ -38,5 +38,5 @@ func ApplyFamilies(ix *Index, families []Family) ([]string, error) {
 		})
 	}
 
-	return galleryedit.Apply(ix.Lines, inserts)
+	return galleryedit.ApplyLibrary(ix.Lines, inserts)
 }

.github/ci/variantproposals/edit_test.go

@@ -33,45 +33,54 @@ var _ = Describe("ApplyFamilies", func() {
 		return n
 	}
 
-	It("adds a variants block right after the entry's name and touches nothing else", func() {
+	It("adds a variants block at the end of the model body and touches nothing else", func() {
 		ix := indexOf(
 			entryYAML("foo-model", "acme/repo", "foo-model-Q4_K_M.gguf", "aa"),
 			entryYAML("foo-model-q8_0", "acme/repo", "foo-model-Q8_0.gguf", "bb"),
 		)
 		out := apply(ix, []Family{{Parent: "foo-model", Proposals: []Proposal{{Variant: "foo-model-q8_0"}}}})
 
-		Expect(out[0]).To(Equal("- name: foo-model"))
-		Expect(out[1]).To(Equal("  variants:"))
-		Expect(out[2]).To(Equal("    - model: foo-model-q8_0"))
+		end := ix.Entries[0].EndLine
+		Expect(out[0]).To(Equal("- kind: model"))
+		Expect(out[end-1]).To(Equal(ix.Lines[end-1]))
+		Expect(out[end]).To(Equal("    variants:"))
+		Expect(out[end+1]).To(Equal("      - model: foo-model-q8_0"))
+		Expect(out[end+2]).To(Equal("- kind: model"))
 		Expect(len(out)).To(Equal(len(ix.Lines) + 2))
 		Expect(insertedLines(ix.Lines, out)).To(Equal(2))
 	})
 
 	It("appends to a variants block that already exists", func() {
-		ix := indexOf(`- name: partial
-  variants:
-    - model: partial-q8_0
-  url: u
-  overrides:
-    parameters:
-      model: partial-Q4_K_M.gguf
+		ix := indexOf(`- kind: model
+  id: partial
+  name: partial
+  model:
+    variants:
+      - model: partial-q8_0
+    url: u
+    overrides:
+      parameters:
+        model: partial-Q4_K_M.gguf
 `, entryYAML("partial-f16", "acme/repo", "partial-f16.gguf", "cc"))
 		out := apply(ix, []Family{{Parent: "partial", Proposals: []Proposal{{Variant: "partial-f16"}}}})
 
-		Expect(out[1]).To(Equal("  variants:"))
-		Expect(out[2]).To(Equal("    - model: partial-q8_0"))
-		Expect(out[3]).To(Equal("    - model: partial-f16"))
-		Expect(out[4]).To(Equal("  url: u"))
+		Expect(out[4]).To(Equal("    variants:"))
+		Expect(out[5]).To(Equal("      - model: partial-q8_0"))
+		Expect(out[6]).To(Equal("      - model: partial-f16"))
+		Expect(out[7]).To(Equal("    url: u"))
 	})
 
 	It("replaces an explicit empty list rather than leaving two variants keys", func() {
-		ix := indexOf(`- name: emptied
-  variants: []
-  url: u
+		ix := indexOf(`- kind: model
+  id: emptied
+  name: emptied
+  model:
+    variants: []
+    url: u
 `, entryYAML("emptied-q8_0", "acme/repo", "emptied-Q8_0.gguf", "cc"))
 		out := apply(ix, []Family{{Parent: "emptied", Proposals: []Proposal{{Variant: "emptied-q8_0"}}}})
 
-		Expect(strings.Join(out[:4], "\n")).To(Equal("- name: emptied\n  variants:\n    - model: emptied-q8_0\n  url: u"))
+		Expect(strings.Join(out[3:7], "\n")).To(Equal("  model:\n    variants:\n      - model: emptied-q8_0\n    url: u"))
 		Expect(strings.Count(strings.Join(out, "\n"), "variants:")).To(Equal(1))
 	})
 
@@ -81,7 +90,7 @@ var _ = Describe("ApplyFamilies", func() {
 			entryYAML("phi-2-chat:Q8_0", "acme/repo", "phi-2-chat-Q8_0.gguf", "bb"),
 		)
 		out := apply(ix, []Family{{Parent: "phi-2-chat", Proposals: []Proposal{{Variant: "phi-2-chat:Q8_0"}}}})
-		Expect(out[2]).To(Equal(`    - model: "phi-2-chat:Q8_0"`))
+		Expect(out[ix.Entries[0].EndLine+1]).To(Equal(`      - model: "phi-2-chat:Q8_0"`))
 
 		// The result has to still be a gallery, and the reference has to
 		// resolve to the entry it names.
@@ -119,35 +128,36 @@ var _ = Describe("ApplyFamilies", func() {
 })
 
 var _ = Describe("ParseIndex", func() {
-	It("records the anchor an entry defines and the anchor an entry merges", func() {
-		ix := indexOf(`- &anc
-  name: anchored
-  url: u
-`, `- !!merge <<: *anc
-  name: child
-`)
-		Expect(ix.Entries[0].AnchorName).To(Equal("anc"))
-		Expect(ix.Entries[1].MergesFrom).To(Equal("anc"))
-		Expect(ix.MergeChildren("anc")).To(HaveLen(1))
-	})
-
-	It("carries merged values into the child, so an inherited variants key is visible", func() {
-		ix := indexOf(`- &anc
-  name: anchored
-  url: u
-  variants:
-    - model: something
-`, `- !!merge <<: *anc
-  name: child
-`)
-		Expect(ix.Entries[1].HasVariants()).To(BeTrue())
-	})
-
 	It("refuses a list item that decodes to nothing", func() {
 		// Every line number the editor works from comes from pairing decoded
 		// entries with top level list items. If those two views can disagree,
 		// the editor writes into the wrong entry, so the parse refuses instead.
-		_, err := ParseIndex("- name: one\n  url: u\n-\n")
+		_, err := ParseIndex("- kind: model\n  id: one\n-\n")
 		Expect(err).To(MatchError(ContainSubstring("empty")))
 	})
 })
+
+var _ = Describe("the checked-in model library", func() {
+	It("parses, and a proposal edit keeps it parsing with the same entries", func() {
+		ix, err := LoadIndex("../../../gallery/models/index.yaml")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(ix.Entries).ToNot(BeEmpty())
+		for _, e := range ix.Entries {
+			Expect(e.Name).ToNot(BeEmpty())
+		}
+
+		// Add one variant to the first and the last entry: the first has a
+		// block, the last may have none, and both must stay valid.
+		first, last := ix.Entries[0], ix.Entries[len(ix.Entries)-1]
+		out, err := ApplyFamilies(ix, []Family{
+			{Parent: first.Name, Proposals: []Proposal{{Variant: "x-added-one"}}},
+			{Parent: last.Name, Proposals: []Proposal{{Variant: "x-added-two"}}},
+		})
+		Expect(err).ToNot(HaveOccurred())
+		reparsed, err := ParseIndex(strings.Join(out, "\n"))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(reparsed.Entries).To(HaveLen(len(ix.Entries)))
+		Expect(reparsed.Entries[0].Variants).To(ContainElement(VariantRef{Model: "x-added-one"}))
+		Expect(reparsed.Entries[len(ix.Entries)-1].Variants).To(ContainElement(VariantRef{Model: "x-added-two"}))
+	})
+})

.github/ci/variantproposals/index.go

@@ -24,46 +24,50 @@ type VariantRef struct {
 	Model string `yaml:"model"`
 }
 
-// GalleryEntry is one gallery entry, carrying both the semantics the heuristics need
-// and the text range the editor needs.
+// GalleryEntry is one model library entry, carrying both the semantics the
+// heuristics need and the text range the editor needs. The name is the id of
+// the entry, which is the name the gallery knows the model by.
 //
 // The two views are kept together deliberately. The editor must not round-trip
-// the index through a YAML marshaller: the gallery is 40,000 lines and a
+// the index through a YAML marshaller: the library is 40,000 lines and a
 // reflowed diff cannot be reviewed, which defeats the entire point of a job
 // whose output is a human decision.
 type GalleryEntry struct {
-	Name       string         `yaml:"name"`
-	URL        string         `yaml:"url"`
-	ConfigFile map[string]any `yaml:"config_file"`
-	Overrides  map[string]any `yaml:"overrides"`
-	Files      []File         `yaml:"files"`
-	Variants   []VariantRef   `yaml:"variants"`
+	Name       string
+	URL        string
+	ConfigFile map[string]any
+	Overrides  map[string]any
+	Files      []File
+	Variants   []VariantRef
 
-	// Index is the entry's position in gallery order.
-	Index int `yaml:"-"`
+	// Index is the entry's position in library order.
+	Index int
 	// StartLine and EndLine bound the entry's lines, zero based and half open.
-	StartLine int `yaml:"-"`
-	EndLine   int `yaml:"-"`
-	// AnchorName is set when the entry defines a YAML anchor. Adding a variants
-	// key to such an entry is inherited by everything that merges it, which is
-	// why proposals involving anchors get special treatment.
-	AnchorName string `yaml:"-"`
-	// MergesFrom is the anchor this entry pulls in with "!!merge <<:".
-	MergesFrom string `yaml:"-"`
+	StartLine int
+	EndLine   int
 }
 
-// Index is a parsed gallery index: entries plus the exact lines they came from.
+// libraryEntry is the subset of a model library entry the proposer decodes.
+// Everything but the id lives under model: in the library.
+type libraryEntry struct {
+	Kind  string `yaml:"kind"`
+	ID    string `yaml:"id"`
+	Model struct {
+		URL        string         `yaml:"url"`
+		ConfigFile map[string]any `yaml:"configFile"`
+		Overrides  map[string]any `yaml:"overrides"`
+		Files      []File         `yaml:"files"`
+		Variants   []VariantRef   `yaml:"variants"`
+	} `yaml:"model"`
+}
+
+// Index is a parsed model library: entries plus the exact lines they came from.
 type Index struct {
 	Lines   []string
 	Entries []*GalleryEntry
 }
 
-var (
-	anchorStart = regexp.MustCompile(`^- &(\S+)`)
-	mergeStart  = regexp.MustCompile(`^- !!merge <<: \*(\S+)`)
-)
-
-// LoadIndex reads and parses a gallery index file.
+// LoadIndex reads and parses a model library file.
 func LoadIndex(path string) (*Index, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -72,55 +76,51 @@ func LoadIndex(path string) (*Index, error) {
 	return ParseIndex(string(data))
 }
 
-// ParseIndex builds an Index from the raw text of a gallery index.
+// ParseIndex builds an Index from the raw text of a model library.
 //
 // The YAML decode and the textual scan are cross checked against each other: if
 // they disagree on how many entries there are, every line number the editor
 // would use is suspect, so the run fails rather than editing the wrong entry.
 func ParseIndex(text string) (*Index, error) {
-	var entries []*GalleryEntry
-	if err := yaml.Unmarshal([]byte(text), &entries); err != nil {
-		return nil, fmt.Errorf("decoding gallery index: %w", err)
+	var decoded []*libraryEntry
+	if err := yaml.Unmarshal([]byte(text), &decoded); err != nil {
+		return nil, fmt.Errorf("decoding model library: %w", err)
 	}
 
 	lines, starts := galleryedit.Scan(text)
-	if len(starts) != len(entries) {
-		return nil, fmt.Errorf("gallery index has %d decoded entries but %d top level list items; refusing to edit by line number", len(entries), len(starts))
+	if len(starts) != len(decoded) {
+		return nil, fmt.Errorf("model library has %d decoded entries but %d top level list items; refusing to edit by line number", len(decoded), len(starts))
 	}
 
-	for i, e := range entries {
-		if e == nil {
-			return nil, fmt.Errorf("gallery index list item %d is empty; refusing to edit by line number", i)
+	entries := make([]*GalleryEntry, 0, len(decoded))
+	for i, d := range decoded {
+		if d == nil {
+			return nil, fmt.Errorf("model library list item %d is empty; refusing to edit by line number", i)
+		}
+		if d.Kind != "model" {
+			return nil, fmt.Errorf("model library list item %d is of kind %q; this job only reads model entries", i, d.Kind)
+		}
+		e := &GalleryEntry{
+			Name:       d.ID,
+			URL:        d.Model.URL,
+			ConfigFile: d.Model.ConfigFile,
+			Overrides:  d.Model.Overrides,
+			Files:      d.Model.Files,
+			Variants:   d.Model.Variants,
+			Index:      i,
+			StartLine:  starts[i],
 		}
-		e.Index = i
-		e.StartLine = starts[i]
 		if i+1 < len(starts) {
 			e.EndLine = starts[i+1]
 		} else {
 			e.EndLine = len(lines)
 		}
-		if m := anchorStart.FindStringSubmatch(lines[e.StartLine]); m != nil {
-			e.AnchorName = m[1]
-		}
-		if m := mergeStart.FindStringSubmatch(lines[e.StartLine]); m != nil {
-			e.MergesFrom = m[1]
-		}
+		entries = append(entries, e)
 	}
 
 	return &Index{Lines: lines, Entries: entries}, nil
 }
 
-// MergeChildren lists the entries that pull in the given anchor.
-func (ix *Index) MergeChildren(anchor string) []*GalleryEntry {
-	var out []*GalleryEntry
-	for _, e := range ix.Entries {
-		if e.MergesFrom == anchor {
-			out = append(out, e)
-		}
-	}
-	return out
-}
-
 // ByName indexes entries by lowercased name. A name appearing twice keeps the
 // first occurrence, matching the gallery's own first-match-wins resolution, and
 // the duplicates are returned so the caller can refuse to touch them: a

.github/ci/variantproposals/main.go

@@ -16,10 +16,10 @@ import (
 )
 
 func main() {
-	index := flag.String("index", "gallery/index.yaml", "path to the gallery index")
+	index := flag.String("index", "gallery/models/index.yaml", "path to the model library")
 	ledger := flag.String("ledger", "gallery/variant-exclusions.yaml", "path to the rejection ledger")
 	bodyOut := flag.String("body-out", "", "write the pull request body here")
-	apply := flag.Bool("apply", false, "write the proposed groupings back into the index")
+	apply := flag.Bool("apply", false, "write the proposed groupings back into the library")
 	flag.Parse()
 
 	if err := run(*index, *ledger, *bodyOut, *apply); err != nil {

.github/ci/variantproposals/propose.go

@@ -407,25 +407,6 @@ func buildFamily(ix *Index, members []int, evidenceFor map[[2]int]Evidence, clai
 		return nil, &Refusal{Members: names, Reason: fmt.Sprintf("the natural parent %q is already proposed as a variant of %q; proposing it as a parent would nest variants", parentEntry.Name, owner)}
 	}
 
-	// Adding a variants key to an anchor is inherited by every entry that
-	// merges it, silently grouping models nobody proposed. Handling that means
-	// editing each merging child too, which is a larger change than this job
-	// should make unsupervised, so it refuses and hands the reviewer the list.
-	if parentEntry.AnchorName != "" {
-		children := ix.MergeChildren(parentEntry.AnchorName)
-		if len(children) > 0 {
-			childNames := make([]string, 0, len(children))
-			for _, c := range children {
-				childNames = append(childNames, c.Name)
-			}
-			return nil, &Refusal{
-				Members: names,
-				Reason: fmt.Sprintf("the parent %q defines YAML anchor &%s, and a variants key added there is inherited by the %d entries that merge it (%s). Grouping this family by hand also means adding an explicit `variants: []` to each of those entries",
-					parentEntry.Name, parentEntry.AnchorName, len(children), strings.Join(childNames, ", ")),
-			}
-		}
-	}
-
 	existing := map[string]struct{}{}
 	for _, v := range parentEntry.Variants {
 		existing[strings.ToLower(v.Model)] = struct{}{}
@@ -445,7 +426,7 @@ func buildFamily(ix *Index, members []int, evidenceFor map[[2]int]Evidence, clai
 			return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q already offers variants of its own, so it cannot itself be a variant target", target.Name)}
 		}
 		if !target.Installable() {
-			return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q has no url, config_file, overrides or files, so it is not independently installable", target.Name)}
+			return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q has no url, configFile, overrides or files, so it is not independently installable", target.Name)}
 		}
 		if owner, claimed := claimedBy[lower]; claimed && owner != parentName {
 			return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q is already a variant of %q; a target claimed by two parents is not something the gallery resolves predictably", target.Name, owner)}

.github/ci/variantproposals/propose_test.go

@@ -8,20 +8,23 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-// entryYAML writes one gallery entry with a single weight file, which is the
+// entryYAML writes one library entry with a single weight file, which is the
 // shape almost every real entry has. Specs that need something else write the
 // YAML out by hand.
 func entryYAML(name, repo, filename, sha string) string {
-	return fmt.Sprintf(`- name: %s
-  url: github:mudler/LocalAI/gallery/virtual.yaml@master
-  overrides:
-    parameters:
-      model: %s
-  files:
-    - filename: %s
-      uri: huggingface://%s/%s
-      sha256: %s
-`, name, filename, filename, repo, filename, sha)
+	return fmt.Sprintf(`- kind: model
+  id: %s
+  name: %s
+  model:
+    url: github:mudler/LocalAI/gallery/virtual.yaml@master
+    overrides:
+      parameters:
+        model: %s
+    files:
+      - filename: %s
+        uri: huggingface://%s/%s
+        sha256: %s
+`, name, name, filename, filename, repo, filename, sha)
 }
 
 func indexOf(entries ...string) *Index {
@@ -103,24 +106,30 @@ var _ = Describe("Propose", func() {
 			// Both entries ship the same text encoder. That is a packaging
 			// convention, not evidence of shared weights: this is how an
 			// earlier sweep linked four wan-2.1 entries to each other.
-			ix := indexOf(`- name: wan-2.1-t2v
-  url: u
-  files:
-    - filename: wan-2.1-t2v-Q4_K_M.gguf
-      uri: huggingface://acme/wan/wan-2.1-t2v-Q4_K_M.gguf
-      sha256: aa
-    - filename: umt5-xxl-encoder-Q8_0.gguf
-      uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf
-      sha256: cc
-`, `- name: z-image-turbo
-  url: u
-  files:
-    - filename: z-image-turbo-Q4_K_M.gguf
-      uri: huggingface://acme/wan/z-image-turbo-Q4_K_M.gguf
-      sha256: bb
-    - filename: umt5-xxl-encoder-Q8_0.gguf
-      uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf
-      sha256: cc
+			ix := indexOf(`- kind: model
+  id: wan-2.1-t2v
+  name: wan-2.1-t2v
+  model:
+    url: u
+    files:
+      - filename: wan-2.1-t2v-Q4_K_M.gguf
+        uri: huggingface://acme/wan/wan-2.1-t2v-Q4_K_M.gguf
+        sha256: aa
+      - filename: umt5-xxl-encoder-Q8_0.gguf
+        uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf
+        sha256: cc
+`, `- kind: model
+  id: z-image-turbo
+  name: z-image-turbo
+  model:
+    url: u
+    files:
+      - filename: z-image-turbo-Q4_K_M.gguf
+        uri: huggingface://acme/wan/z-image-turbo-Q4_K_M.gguf
+        sha256: bb
+      - filename: umt5-xxl-encoder-Q8_0.gguf
+        uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf
+        sha256: cc
 `)
 			r := Propose(ix, nil)
 			Expect(familyNames(r)).To(BeEmpty())
@@ -162,16 +171,17 @@ var _ = Describe("Propose", func() {
 		It("does not group entries with a byte-identical install payload", func() {
 			// whisper-1 exists so OpenAI-compatible clients can send that name.
 			// Folding it under whisper-base would hide the name they send.
-			payload := `  url: github:mudler/LocalAI/gallery/whisper-base.yaml@master
-  overrides:
-    parameters:
-      model: ggml-whisper-base.bin
-  files:
-    - filename: ggml-whisper-base.bin
-      uri: huggingface://ggerganov/whisper.cpp/ggml-base.bin
-      sha256: aa
+			payload := `  model:
+    url: github:mudler/LocalAI/gallery/whisper-base.yaml@master
+    overrides:
+      parameters:
+        model: ggml-whisper-base.bin
+    files:
+      - filename: ggml-whisper-base.bin
+        uri: huggingface://ggerganov/whisper.cpp/ggml-base.bin
+        sha256: aa
 `
-			ix := indexOf("- name: whisper-base\n"+payload, "- name: whisper-1\n"+payload)
+			ix := indexOf("- kind: model\n  id: whisper-base\n"+payload, "- kind: model\n  id: whisper-1\n"+payload)
 			r := Propose(ix, nil)
 			Expect(familyNames(r)).To(BeEmpty())
 			Expect(r.AliasSkipped).To(HaveLen(1))
@@ -256,17 +266,20 @@ var _ = Describe("Propose", func() {
 		It("refuses to nest: a target that already offers variants of its own", func() {
 			ix := indexOf(
 				entryYAML("nest-model", "acme/repo", "nest-model-Q4_K_M.gguf", "aa"),
-				`- name: nest-model-q8_0
-  url: u
-  variants:
-    - model: nest-model-q8_0-mtp
-  overrides:
-    parameters:
-      model: nest-model-Q8_0.gguf
-  files:
-    - filename: nest-model-Q8_0.gguf
-      uri: huggingface://acme/repo/nest-model-Q8_0.gguf
-      sha256: bb
+				`- kind: model
+  id: nest-model-q8_0
+  name: nest-model-q8_0
+  model:
+    url: u
+    variants:
+      - model: nest-model-q8_0-mtp
+    overrides:
+      parameters:
+        model: nest-model-Q8_0.gguf
+    files:
+      - filename: nest-model-Q8_0.gguf
+        uri: huggingface://acme/repo/nest-model-Q8_0.gguf
+        sha256: bb
 `,
 				entryYAML("nest-model-q8_0-mtp", "other/repo", "nest-model-mtp.gguf", "cc"),
 			)
@@ -277,17 +290,20 @@ var _ = Describe("Propose", func() {
 
 		It("refuses to nest: a parent that is already somebody else's variant", func() {
 			ix := indexOf(
-				`- name: outer
-  url: u
-  variants:
-    - model: middle
-  overrides:
-    parameters:
-      model: outer-Q4_K_M.gguf
-  files:
-    - filename: outer-Q4_K_M.gguf
-      uri: huggingface://acme/repo/outer-Q4_K_M.gguf
-      sha256: aa
+				`- kind: model
+  id: outer
+  name: outer
+  model:
+    url: u
+    variants:
+      - model: middle
+    overrides:
+      parameters:
+        model: outer-Q4_K_M.gguf
+    files:
+      - filename: outer-Q4_K_M.gguf
+        uri: huggingface://acme/repo/outer-Q4_K_M.gguf
+        sha256: aa
 `,
 				entryYAML("middle", "acme/other", "middle-Q4_K_M.gguf", "bb"),
 				entryYAML("middle-q8_0", "acme/other", "middle-Q8_0.gguf", "cc"),
@@ -299,17 +315,20 @@ var _ = Describe("Propose", func() {
 
 		It("refuses to let two parents claim one target", func() {
 			ix := indexOf(
-				`- name: claimant
-  url: u
-  variants:
-    - model: contested-q8_0
-  overrides:
-    parameters:
-      model: claimant-Q4_K_M.gguf
-  files:
-    - filename: claimant-Q4_K_M.gguf
-      uri: huggingface://acme/repo/claimant-Q4_K_M.gguf
-      sha256: aa
+				`- kind: model
+  id: claimant
+  name: claimant
+  model:
+    url: u
+    variants:
+      - model: contested-q8_0
+    overrides:
+      parameters:
+        model: claimant-Q4_K_M.gguf
+    files:
+      - filename: claimant-Q4_K_M.gguf
+        uri: huggingface://acme/repo/claimant-Q4_K_M.gguf
+        sha256: aa
 `,
 				entryYAML("contested", "acme/other", "contested-Q4_K_M.gguf", "bb"),
 				entryYAML("contested-q8_0", "acme/other", "contested-Q8_0.gguf", "cc"),
@@ -322,46 +341,13 @@ var _ = Describe("Propose", func() {
 		It("refuses a target that is not independently installable", func() {
 			ix := indexOf(
 				entryYAML("stub-model", "acme/repo", "stub-model-Q4_K_M.gguf", "aa"),
-				"- name: stub-model-q8_0\n  description: a stanza nobody finished\n",
+				"- kind: model\n  id: stub-model-q8_0\n  name: stub-model-q8_0\n  description: a stanza nobody finished\n  model: {}\n",
 			)
 			r := Propose(ix, nil)
 			Expect(familyNames(r)).To(BeEmpty())
 			Expect(refusalReasons(r)).To(ContainSubstring("not independently installable"))
 		})
 
-		It("refuses a family whose parent defines a merge anchor, naming the entries that would inherit", func() {
-			ix := indexOf(
-				`- &anchored
-  name: anchored-model
-  url: u
-  overrides:
-    parameters:
-      model: anchored-Q4_K_M.gguf
-  files:
-    - filename: anchored-Q4_K_M.gguf
-      uri: huggingface://acme/repo/anchored-Q4_K_M.gguf
-      sha256: aa
-`,
-				`- !!merge <<: *anchored
-  name: anchored-child
-  variants: []
-  overrides:
-    parameters:
-      model: unrelated-child-Q4_K_M.gguf
-  files:
-    - filename: unrelated-child-Q4_K_M.gguf
-      uri: huggingface://other/repo/unrelated-child-Q4_K_M.gguf
-      sha256: cc
-`,
-				entryYAML("anchored-model-q8_0", "acme/repo", "anchored-Q8_0.gguf", "bb"),
-			)
-			r := Propose(ix, nil)
-			Expect(familyNames(r)).To(BeEmpty())
-			Expect(refusalReasons(r)).To(ContainSubstring("defines YAML anchor &anchored"))
-			Expect(refusalReasons(r)).To(ContainSubstring("anchored-child"))
-			Expect(refusalReasons(r)).To(ContainSubstring("variants: []"))
-		})
-
 		It("refuses an entry whose name is not unique in the gallery", func() {
 			ix := indexOf(
 				entryYAML("twin", "acme/repo", "twin-Q4_K_M.gguf", "aa"),
@@ -375,17 +361,20 @@ var _ = Describe("Propose", func() {
 
 		It("says nothing about a pair that is already grouped", func() {
 			ix := indexOf(
-				`- name: settled
-  url: u
-  variants:
-    - model: settled-q8_0
-  overrides:
-    parameters:
-      model: settled-Q4_K_M.gguf
-  files:
-    - filename: settled-Q4_K_M.gguf
-      uri: huggingface://acme/repo/settled-Q4_K_M.gguf
-      sha256: aa
+				`- kind: model
+  id: settled
+  name: settled
+  model:
+    url: u
+    variants:
+      - model: settled-q8_0
+    overrides:
+      parameters:
+        model: settled-Q4_K_M.gguf
+    files:
+      - filename: settled-Q4_K_M.gguf
+        uri: huggingface://acme/repo/settled-Q4_K_M.gguf
+        sha256: aa
 `,
 				entryYAML("settled-q8_0", "acme/repo", "settled-Q8_0.gguf", "bb"),
 			)
@@ -397,17 +386,20 @@ var _ = Describe("Propose", func() {
 
 		It("adds only the missing members to a family that already exists", func() {
 			ix := indexOf(
-				`- name: partial
-  url: u
-  variants:
-    - model: partial-q8_0
-  overrides:
-    parameters:
-      model: partial-Q4_K_M.gguf
-  files:
-    - filename: partial-Q4_K_M.gguf
-      uri: huggingface://acme/repo/partial-Q4_K_M.gguf
-      sha256: aa
+				`- kind: model
+  id: partial
+  name: partial
+  model:
+    url: u
+    variants:
+      - model: partial-q8_0
+    overrides:
+      parameters:
+        model: partial-Q4_K_M.gguf
+    files:
+      - filename: partial-Q4_K_M.gguf
+        uri: huggingface://acme/repo/partial-Q4_K_M.gguf
+        sha256: aa
 `,
 				entryYAML("partial-q8_0", "acme/repo", "partial-Q8_0.gguf", "bb"),
 				entryYAML("partial-f16", "acme/repo", "partial-f16.gguf", "cc"),

.github/gallery-agent/gallery.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	"github.com/mudler/LocalAI/core/gallery/importers"
@@ -155,59 +154,32 @@ func generateYAMLEntry(model ProcessedModel, quantization string) string {
 	)
 }
 
-// generateYAMLForModels generates YAML entries for selected models and appends to index.yaml
+// generateYAMLForModels generates library entries for the selected models and
+// appends them to the model library, gallery/models/index.yaml.
+//
+// The entry text is still built in the gallery shape, because that is what the
+// importer output maps onto. appendLibraryEntries converts each entry to the
+// library shape before anything is written.
 func generateYAMLForModels(ctx context.Context, models []ProcessedModel, quantization string) error {
-
-	// Generate YAML entries for each model
 	var yamlEntries []string
 	for _, model := range models {
 		fmt.Printf("Generating YAML entry for model: %s\n", model.ModelID)
-
-		// Generate YAML entry
-		yamlEntry := generateYAMLEntry(model, quantization)
-		yamlEntries = append(yamlEntries, yamlEntry)
+		yamlEntries = append(yamlEntries, generateYAMLEntry(model, quantization))
 	}
 
-	// Prepend to index.yaml (write at the top)
-	if len(yamlEntries) > 0 {
-		indexPath := getGalleryIndexPath()
-		fmt.Printf("Prepending YAML entries to %s...\n", indexPath)
-
-		// Read current content
-		content, err := os.ReadFile(indexPath)
-		if err != nil {
-			return fmt.Errorf("failed to read %s: %w", indexPath, err)
-		}
-
-		existingContent := string(content)
-		yamlBlock := strings.Join(yamlEntries, "\n")
-
-		// Check if file starts with "---"
-		var newContent string
-		if strings.HasPrefix(existingContent, "---\n") {
-			// File starts with "---", prepend new entries after it
-			restOfContent := strings.TrimPrefix(existingContent, "---\n")
-			// Ensure proper spacing: "---\n" + new entries + "\n" + rest of content
-			newContent = "---\n" + yamlBlock + "\n" + restOfContent
-		} else if strings.HasPrefix(existingContent, "---") {
-			// File starts with "---" but no newline after
-			restOfContent := strings.TrimPrefix(existingContent, "---")
-			newContent = "---\n" + yamlBlock + "\n" + strings.TrimPrefix(restOfContent, "\n")
-		} else {
-			// No "---" at start, prepend new entries at the very beginning
-			// Trim leading whitespace from existing content
-			existingContent = strings.TrimLeft(existingContent, " \t\n\r")
-			newContent = yamlBlock + "\n" + existingContent
-		}
-
-		// Write back to file
-		err = os.WriteFile(indexPath, []byte(newContent), 0644)
-		if err != nil {
-			return fmt.Errorf("failed to write %s: %w", indexPath, err)
-		}
-
-		fmt.Printf("Successfully prepended %d models to %s\n", len(yamlEntries), indexPath)
+	if len(yamlEntries) == 0 {
+		return nil
 	}
 
+	indexPath := getGalleryIndexPath()
+	fmt.Printf("Appending library entries to %s...\n", indexPath)
+	added, skipped, err := appendLibraryEntries(indexPath, yamlEntries)
+	if err != nil {
+		return err
+	}
+	for _, id := range skipped {
+		fmt.Printf("Skipped %s: the library already has an entry with this id\n", id)
+	}
+	fmt.Printf("Successfully appended %d models to %s\n", len(added), indexPath)
 	return nil
 }

.github/gallery-agent/helpers.go

@@ -13,14 +13,19 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+// A library entry has the same top-level name and urls keys as a gallery
+// entry, so galleryModel below reads both.
+
 var galleryIndexPath = os.Getenv("GALLERY_INDEX_PATH")
 
-// getGalleryIndexPath returns the gallery index file path, with a default fallback
+// getGalleryIndexPath returns the model library path, with a default fallback.
+// The library, gallery/models/index.yaml, is the source of the models; the
+// gallery/index.yaml copy is generated from it and is never written here.
 func getGalleryIndexPath() string {
 	if galleryIndexPath != "" {
 		return galleryIndexPath
 	}
-	return "gallery/index.yaml"
+	return "gallery/models/index.yaml"
 }
 
 type galleryModel struct {
@@ -28,7 +33,7 @@ type galleryModel struct {
 	Urls []string `yaml:"urls"`
 }
 
-// loadGalleryURLSet parses gallery/index.yaml once and returns the set of
+// loadGalleryURLSet parses the model library once and returns the set of
 // HuggingFace model URLs already present in the gallery.
 func loadGalleryURLSet() (map[string]struct{}, error) {
 	indexPath := getGalleryIndexPath()
@@ -142,7 +147,7 @@ func resolveReadme(client *hfapi.Client, modelID string, hfTags []string) (strin
 }
 
 // extractDescription turns a raw HuggingFace README into a concise plain-text
-// description suitable for embedding in gallery/index.yaml: strips YAML
+// description suitable for embedding in a gallery entry: strips YAML
 // frontmatter, HTML tags/comments, markdown images, link URLs (keeping the
 // link text), markdown tables, and then truncates at a paragraph boundary
 // around ~1200 characters. Raw README should still be used for icon

.github/gallery-agent/library.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"gopkg.in/yaml.v3"
+)
+
+// appendLibraryEntries converts gallery-shaped entry texts to model library
+// entries and appends them to the library file at path. It returns the ids it
+// added and the ids it skipped because the library already has them.
+//
+// The entries are appended as text, after the existing content, so the diff of
+// the library is only the new entries. The result is decoded again with the
+// reader the host uses before it is written, so a malformed entry fails the
+// run instead of reaching a pull request.
+func appendLibraryEntries(path string, galleryEntries []string) (added, skipped []string, err error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to read %s: %w", path, err)
+	}
+	current, err := gallery.DecodeModelIndex(content)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to decode %s as a model library: %w", path, err)
+	}
+	ids := make(map[string]struct{}, len(current.Entries))
+	for _, e := range current.Entries {
+		ids[e.ID] = struct{}{}
+	}
+
+	var entries []gallery.ModelLibraryEntry
+	for _, text := range galleryEntries {
+		var models []gallery.GalleryModel
+		if err := yaml.Unmarshal([]byte(text), &models); err != nil {
+			return nil, nil, fmt.Errorf("generated entry does not decode: %w", err)
+		}
+		for _, m := range models {
+			entry, err := gallery.ModelLibraryEntryFromGallery(m)
+			if err != nil {
+				return nil, nil, fmt.Errorf("converting %q to a library entry: %w", m.Name, err)
+			}
+			if _, dup := ids[entry.ID]; dup {
+				skipped = append(skipped, entry.ID)
+				continue
+			}
+			ids[entry.ID] = struct{}{}
+			entries = append(entries, entry)
+			added = append(added, entry.ID)
+		}
+	}
+	if len(entries) == 0 {
+		return nil, skipped, nil
+	}
+
+	encoded, err := gallery.EncodeModelIndex(entries)
+	if err != nil {
+		return nil, nil, fmt.Errorf("encoding library entries: %w", err)
+	}
+
+	out := string(content)
+	if !strings.HasSuffix(out, "\n") {
+		out += "\n"
+	}
+	out += string(encoded)
+	if _, err := gallery.DecodeModelIndex([]byte(out)); err != nil {
+		return nil, nil, fmt.Errorf("the library with the new entries does not decode: %w", err)
+	}
+	if err := os.WriteFile(path, []byte(out), 0o644); err != nil {
+		return nil, nil, fmt.Errorf("failed to write %s: %w", path, err)
+	}
+	return added, skipped, nil
+}

.github/gallery-agent/library_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mudler/LocalAI/core/gallery"
+)
+
+const libraryFixture = `# header comment
+- kind: model
+  id: existing-model
+  name: existing-model
+  version: 0.0.0-gallery.000000000000
+  description: |
+    An existing model.
+  urls:
+    - https://huggingface.co/acme/existing-GGUF
+  channel: gallery
+  schemaVersion: 1
+  model:
+    url: github:mudler/LocalAI/gallery/virtual.yaml@master
+`
+
+// agentEntry has the shape generateYAMLEntry writes.
+const agentEntry = `- name: "new-model"
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/acme/new-model-GGUF
+  description: |
+    A new model.
+  license: "apache-2.0"
+  tags:
+    - llm
+    - gguf
+  overrides:
+    parameters:
+      model: new-model-Q4_K_M.gguf
+  files:
+    - filename: new-model-Q4_K_M.gguf
+      sha256: abc
+      uri: huggingface://acme/new-model-GGUF/new-model-Q4_K_M.gguf
+`
+
+func writeFixture(t *testing.T) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), "index.yaml")
+	if err := os.WriteFile(p, []byte(libraryFixture), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+func TestAppendLibraryEntriesWritesTheLibraryShape(t *testing.T) {
+	p := writeFixture(t)
+	added, skipped, err := appendLibraryEntries(p, []string{agentEntry})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(added) != 1 || added[0] != "new-model" || len(skipped) != 0 {
+		t.Fatalf("added %v skipped %v", added, skipped)
+	}
+
+	got, _ := os.ReadFile(p)
+	if !strings.HasPrefix(string(got), libraryFixture) {
+		t.Fatal("existing content was changed; entries must only be appended")
+	}
+	res, err := gallery.DecodeModelIndex(got)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res.Entries) != 2 {
+		t.Fatalf("want 2 entries, got %d", len(res.Entries))
+	}
+	e := res.Entries[1]
+	if e.ID != "new-model" || e.Kind != "model" {
+		t.Fatalf("unexpected entry %+v", e.Envelope)
+	}
+	if e.Model.URL == "" || len(e.Model.Files) != 1 || e.Model.Overrides == nil {
+		t.Fatalf("model body not filled: %+v", e.Model)
+	}
+	if len(e.URLs) != 1 || e.URLs[0] != "https://huggingface.co/acme/new-model-GGUF" {
+		t.Fatalf("urls: %v", e.URLs)
+	}
+}
+
+func TestAppendLibraryEntriesSkipsAnExistingID(t *testing.T) {
+	p := writeFixture(t)
+	dup := strings.Replace(agentEntry, `"new-model"`, `"existing-model"`, 1)
+	added, skipped, err := appendLibraryEntries(p, []string{dup})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(added) != 0 || len(skipped) != 1 {
+		t.Fatalf("added %v skipped %v", added, skipped)
+	}
+	got, _ := os.ReadFile(p)
+	if string(got) != libraryFixture {
+		t.Fatal("file changed although nothing was added")
+	}
+}
+
+func TestDuplicateCheckReadsTheLibrary(t *testing.T) {
+	p := writeFixture(t)
+	if _, _, err := appendLibraryEntries(p, []string{agentEntry}); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("GALLERY_INDEX_PATH", p)
+	old := galleryIndexPath
+	galleryIndexPath = p
+	defer func() { galleryIndexPath = old }()
+
+	set, err := loadGalleryURLSet()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, id := range []string{"acme/existing-GGUF", "acme/new-model-GGUF"} {
+		if !modelAlreadyInGallery(set, id) {
+			t.Fatalf("%s should count as already in the library", id)
+		}
+	}
+}

.github/gallery-agent/testing.go

@@ -8,7 +8,7 @@ import (
 	"time"
 )
 
-// runSyntheticMode generates synthetic test data and appends it to the gallery
+// runSyntheticMode generates synthetic test data and appends it to the model library
 func runSyntheticMode() error {
 	generator := NewSyntheticDataGenerator()
 
@@ -23,7 +23,7 @@ func runSyntheticMode() error {
 		fmt.Printf("Generated synthetic model: %s\n", model.ModelID)
 	}
 
-	// Generate YAML entries and append to gallery/index.yaml
+	// Generate YAML entries and append to the model library
 	fmt.Println("Generating YAML entries for synthetic models...")
 	err := generateYAMLForModels(context.Background(), models, "Q4_K_M")
 	if err != nil {

.github/workflows/checksum_checker.yaml

@@ -16,6 +16,19 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y pip wget
           pip install huggingface_hub
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Generate the gRPC stubs the Go packages import
+        # modelcatalog imports them and they are not in the tree. Same pinned
+        # tools as the model library workflow.
+        run: |
+          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip
+          sudo unzip -j -d /usr/local/bin protoc.zip bin/protoc
+          rm protoc.zip
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          PATH="$PATH:$HOME/go/bin" make protogen-go
       - name: 'Setup yq'
         uses: dcarbone/install-yq-action@v1.3.1
         with:
@@ -28,14 +41,19 @@ jobs:
           export HF_HOME=/hf_cache
           sudo mkdir /hf_cache
           sudo chmod 777 /hf_cache
-          bash .github/checksum_checker.sh gallery/index.yaml
+          # The model library is the source. The checker edits it.
+          bash .github/checksum_checker.sh gallery/models/index.yaml
+      - name: Regenerate the gallery copy from the library
+        # gallery/index.yaml is generated; the model library check fails the
+        # PR when it differs from the generated copy.
+        run: go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ secrets.UPDATE_BOT_TOKEN }}
           push-to-fork: ci-forks/LocalAI
-          commit-message: ':arrow_up: Checksum updates in gallery/index.yaml'
+          commit-message: ':arrow_up: Checksum updates in the model library'
           title: 'chore(model-gallery): :arrow_up: update checksum'
           branch: "update/checksum"
-          body: Updating checksums in gallery/index.yaml
+          body: Updating checksums in gallery/models/index.yaml and regenerating gallery/index.yaml
           signoff: true

.github/workflows/gallery-agent.yaml

@@ -38,7 +38,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version-file: go.mod
       - name: Proto Dependencies
         run: |
           # Install protoc
@@ -131,19 +131,25 @@ jobs:
           MAX_MODELS: ${{ github.event.inputs.max_models || '1' }}
           EXTRA_SKIP_URLS: ${{ steps.open_prs.outputs.urls }}
         run: |
-          export GALLERY_INDEX_PATH=$PWD/gallery/index.yaml
+          # The agent reads and appends to the model library, the source.
+          export GALLERY_INDEX_PATH=$PWD/gallery/models/index.yaml
           go run ./.github/gallery-agent
 
+      - name: Regenerate the gallery copy from the library
+        # gallery/index.yaml is generated from the library; the model library
+        # check fails the PR when they disagree.
+        run: go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml
+
       - name: Check for changes
         id: check_changes
         run: |
-          if git diff --quiet gallery/index.yaml; then
+          if git diff --quiet gallery/models/index.yaml gallery/index.yaml; then
             echo "changes=false" >> $GITHUB_OUTPUT
-            echo "No changes detected in gallery/index.yaml"
+            echo "No changes detected in the model library"
           else
             echo "changes=true" >> $GITHUB_OUTPUT
-            echo "Changes detected in gallery/index.yaml"
-            git diff gallery/index.yaml
+            echo "Changes detected in the model library"
+            git diff gallery/models/index.yaml gallery/index.yaml
           fi
 
       - name: Read gallery agent summary

.github/workflows/gallery_variant_proposals.yaml

@@ -19,12 +19,24 @@ jobs:
       - name: Test the proposer
         run: go test ./.github/ci/variantproposals/
 
+      # The proposer calls no gallery code, but modelcatalog imports packages
+      # that need the generated gRPC stubs. Same pinned tools as the model
+      # library workflow.
+      - name: Generate the gRPC stubs the Go packages import
+        run: |
+          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip
+          sudo unzip -j -d /usr/local/bin protoc.zip bin/protoc
+          rm protoc.zip
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          PATH="$PATH:$HOME/go/bin" make protogen-go
+
       - name: Propose groupings 🔧
         id: propose
         run: |
           rm -f /tmp/variant-proposals-body.md
           go run ./.github/ci/variantproposals \
-            -index gallery/index.yaml \
+            -index gallery/models/index.yaml \
             -ledger gallery/variant-exclusions.yaml \
             -body-out /tmp/variant-proposals-body.md \
             -apply
@@ -39,6 +51,13 @@ jobs:
             echo "have_proposals=false" >> "$GITHUB_OUTPUT"
           fi
 
+      # The proposer edits the model library, the source. The gallery copy is
+      # generated from it, and the model library check fails a PR where they
+      # disagree. Only regenerate when the library changed.
+      - name: Regenerate the gallery copy from the library
+        if: steps.propose.outputs.have_proposals == 'true'
+        run: go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml
+
       # No body file means the proposer found nothing. Opening an empty pull
       # request every run is how a proposal job gets muted by its reviewers.
       - name: Create Pull Request