diff --git a/.github/checksum_checker.sh b/.github/checksum_checker.sh index 5cbd57f4a..3bebf1aa8 100644 --- a/.github/checksum_checker.sh +++ b/.github/checksum_checker.sh @@ -2,7 +2,9 @@ # This scripts needs yq and huggingface_hub to be installed # to install hugingface_hub run pip install huggingface_hub -# Path to the input YAML file +# Path to the input YAML file: the model library, gallery/models/index.yaml. +# Its entries keep the files under .model.files. The caller regenerates the +# gallery copy, gallery/index.yaml, after this script ran. input_yaml=$1 # Function to download file and check checksum using Python @@ -34,11 +36,11 @@ function check_and_update_checksum() { if [[ $result -eq 2 ]]; then echo "File not found, deleting entry for $file_name..." - # yq eval -i "del(.[$idx].files[] | select(.filename == \"$file_name\"))" "$input_yaml" + # yq eval -i "del(.[$idx].model.files[] | select(.filename == \"$file_name\"))" "$input_yaml" elif [[ "$old_checksum" != "$new_checksum" ]]; then echo "Checksum mismatch for $file_name. Updating..." - yq eval -i "del(.[$idx].files[] | select(.filename == \"$file_name\").sha256)" "$input_yaml" - yq eval -i "(.[$idx].files[] | select(.filename == \"$file_name\")).sha256 = \"$new_checksum\"" "$input_yaml" + yq eval -i "del(.[$idx].model.files[] | select(.filename == \"$file_name\").sha256)" "$input_yaml" + yq eval -i "(.[$idx].model.files[] | select(.filename == \"$file_name\")).sha256 = \"$new_checksum\"" "$input_yaml" elif [[ $result -ne 0 ]]; then echo "Error downloading file $file_name. Skipping..." else @@ -51,12 +53,12 @@ len=$(yq eval '. | length' "$input_yaml") for ((i=0; i<$len; i++)) do name=$(yq eval ".[$i].name" "$input_yaml") - files_len=$(yq eval ".[$i].files | length" "$input_yaml") + files_len=$(yq eval ".[$i].model.files | length" "$input_yaml") for ((j=0; j<$files_len; j++)) do - filename=$(yq eval ".[$i].files[$j].filename" "$input_yaml") - uri=$(yq eval ".[$i].files[$j].uri" "$input_yaml") - checksum=$(yq eval ".[$i].files[$j].sha256" "$input_yaml") + filename=$(yq eval ".[$i].model.files[$j].filename" "$input_yaml") + uri=$(yq eval ".[$i].model.files[$j].uri" "$input_yaml") + checksum=$(yq eval ".[$i].model.files[$j].sha256" "$input_yaml") echo "Checking model $name, file $filename. URI = $uri, Checksum = $checksum" check_and_update_checksum "$name" "$filename" "$uri" "$checksum" "$i" done diff --git a/.github/ci/galleryedit/edit.go b/.github/ci/galleryedit/edit.go index 62044d025..22a57e95c 100644 --- a/.github/ci/galleryedit/edit.go +++ b/.github/ci/galleryedit/edit.go @@ -25,6 +25,35 @@ var ( unsafeInName = regexp.MustCompile(`[:#{}\[\],&*?|>'"%@` + "`" + `]|^\s|\s$`) ) +// layout says where the variants key and its items sit in an entry. The +// gallery keeps them at the top of the entry, the model library keeps them in +// the model body. +type layout struct { + keyVariants *regexp.Regexp + variantItem *regexp.Regexp + keyLine string + itemPrefix string + // afterName puts a new variants key right after the name line. Without + // it a new key goes after the last line of the entry, where the library + // keeps the end of the model body. + afterName bool +} + +var galleryLayout = layout{ + keyVariants: keyVariants, + variantItem: variantItem, + keyLine: " variants:", + itemPrefix: " - model: ", + afterName: true, +} + +var libraryLayout = layout{ + keyVariants: regexp.MustCompile(`^ variants:\s*(.*)$`), + variantItem: regexp.MustCompile(`^ - `), + keyLine: " variants:", + itemPrefix: " - model: ", +} + // Entry is the positional view of one gallery entry: what it is called and // which lines it occupies. Nothing about what the entry MEANS belongs here, so // each caller keeps its own semantic decode and only hands over the coordinates. @@ -55,8 +84,19 @@ func Scan(text string) (lines []string, starts []int) { return lines, starts } -// Apply splices every insert into the index lines and returns the new text. +// Apply splices every insert into the gallery index lines and returns the new +// text. func Apply(lines []string, inserts []Insert) ([]string, error) { + return apply(lines, inserts, galleryLayout) +} + +// ApplyLibrary does the same for the model library, gallery/models/index.yaml, +// where an entry keeps its variants under the model: body. +func ApplyLibrary(lines []string, inserts []Insert) ([]string, error) { + return apply(lines, inserts, libraryLayout) +} + +func apply(lines []string, inserts []Insert, l layout) ([]string, error) { type edit struct { at int remove int @@ -71,16 +111,16 @@ func Apply(lines []string, inserts []Insert) ([]string, error) { items := make([]string, 0, len(in.Variants)) for _, v := range in.Variants { - items = append(items, " - model: "+QuoteName(v)) + items = append(items, l.itemPrefix+QuoteName(v)) } - at, remove, err := insertionPoint(lines, in.Entry) + at, remove, err := insertionPoint(lines, in.Entry, l) if err != nil { return nil, err } block := items - if remove > 0 || !hasVariantsKey(lines, in.Entry) { - block = append([]string{" variants:"}, items...) + if remove > 0 || !hasVariantsKey(lines, in.Entry, l) { + block = append([]string{l.keyLine}, items...) } edits = append(edits, edit{at: at, remove: remove, insert: block}) } @@ -97,9 +137,9 @@ func Apply(lines []string, inserts []Insert) ([]string, error) { return out, nil } -func hasVariantsKey(lines []string, e Entry) bool { +func hasVariantsKey(lines []string, e Entry, l layout) bool { for i := e.StartLine; i < e.EndLine; i++ { - if keyVariants.MatchString(lines[i]) { + if l.keyVariants.MatchString(lines[i]) { return true } } @@ -112,9 +152,9 @@ func hasVariantsKey(lines []string, e Entry) bool { // An entry with no variants key gets one right after its name, which is where // the hand-written families put it. An entry with an empty "variants: []" has // that line replaced by a block. An entry with a block gets its items appended. -func insertionPoint(lines []string, e Entry) (at int, remove int, err error) { +func insertionPoint(lines []string, e Entry, l layout) (at int, remove int, err error) { for i := e.StartLine; i < e.EndLine; i++ { - m := keyVariants.FindStringSubmatch(lines[i]) + m := l.keyVariants.FindStringSubmatch(lines[i]) if m == nil { continue } @@ -125,12 +165,19 @@ func insertionPoint(lines []string, e Entry) (at int, remove int, err error) { return 0, 0, fmt.Errorf("entry %q writes its variants inline (%q); this job only edits block lists", e.Name, strings.TrimSpace(m[1])) } last := i - for j := i + 1; j < e.EndLine && variantItem.MatchString(lines[j]); j++ { + for j := i + 1; j < e.EndLine && l.variantItem.MatchString(lines[j]); j++ { last = j } return last + 1, 0, nil } + if !l.afterName { + last := e.EndLine - 1 + for last > e.StartLine && strings.TrimSpace(lines[last]) == "" { + last-- + } + return last + 1, 0, nil + } if inlineName.MatchString(lines[e.StartLine]) { return e.StartLine + 1, 0, nil } diff --git a/.github/ci/variantproposals/edit.go b/.github/ci/variantproposals/edit.go index 47d6e305a..b16ac3c62 100644 --- a/.github/ci/variantproposals/edit.go +++ b/.github/ci/variantproposals/edit.go @@ -7,7 +7,7 @@ import ( "github.com/mudler/LocalAI/.github/ci/galleryedit" ) -// ApplyFamilies writes the proposed variant lists into the index text. +// ApplyFamilies writes the proposed variant lists into the library text. // // The line editing itself lives in galleryedit, shared with the apexentries // generator. Both jobs add variants to entries the gallery already ships, and a @@ -38,5 +38,5 @@ func ApplyFamilies(ix *Index, families []Family) ([]string, error) { }) } - return galleryedit.Apply(ix.Lines, inserts) + return galleryedit.ApplyLibrary(ix.Lines, inserts) } diff --git a/.github/ci/variantproposals/edit_test.go b/.github/ci/variantproposals/edit_test.go index a0312bd51..31243ddfb 100644 --- a/.github/ci/variantproposals/edit_test.go +++ b/.github/ci/variantproposals/edit_test.go @@ -33,45 +33,54 @@ var _ = Describe("ApplyFamilies", func() { return n } - It("adds a variants block right after the entry's name and touches nothing else", func() { + It("adds a variants block at the end of the model body and touches nothing else", func() { ix := indexOf( entryYAML("foo-model", "acme/repo", "foo-model-Q4_K_M.gguf", "aa"), entryYAML("foo-model-q8_0", "acme/repo", "foo-model-Q8_0.gguf", "bb"), ) out := apply(ix, []Family{{Parent: "foo-model", Proposals: []Proposal{{Variant: "foo-model-q8_0"}}}}) - Expect(out[0]).To(Equal("- name: foo-model")) - Expect(out[1]).To(Equal(" variants:")) - Expect(out[2]).To(Equal(" - model: foo-model-q8_0")) + end := ix.Entries[0].EndLine + Expect(out[0]).To(Equal("- kind: model")) + Expect(out[end-1]).To(Equal(ix.Lines[end-1])) + Expect(out[end]).To(Equal(" variants:")) + Expect(out[end+1]).To(Equal(" - model: foo-model-q8_0")) + Expect(out[end+2]).To(Equal("- kind: model")) Expect(len(out)).To(Equal(len(ix.Lines) + 2)) Expect(insertedLines(ix.Lines, out)).To(Equal(2)) }) It("appends to a variants block that already exists", func() { - ix := indexOf(`- name: partial - variants: - - model: partial-q8_0 - url: u - overrides: - parameters: - model: partial-Q4_K_M.gguf + ix := indexOf(`- kind: model + id: partial + name: partial + model: + variants: + - model: partial-q8_0 + url: u + overrides: + parameters: + model: partial-Q4_K_M.gguf `, entryYAML("partial-f16", "acme/repo", "partial-f16.gguf", "cc")) out := apply(ix, []Family{{Parent: "partial", Proposals: []Proposal{{Variant: "partial-f16"}}}}) - Expect(out[1]).To(Equal(" variants:")) - Expect(out[2]).To(Equal(" - model: partial-q8_0")) - Expect(out[3]).To(Equal(" - model: partial-f16")) - Expect(out[4]).To(Equal(" url: u")) + Expect(out[4]).To(Equal(" variants:")) + Expect(out[5]).To(Equal(" - model: partial-q8_0")) + Expect(out[6]).To(Equal(" - model: partial-f16")) + Expect(out[7]).To(Equal(" url: u")) }) It("replaces an explicit empty list rather than leaving two variants keys", func() { - ix := indexOf(`- name: emptied - variants: [] - url: u + ix := indexOf(`- kind: model + id: emptied + name: emptied + model: + variants: [] + url: u `, entryYAML("emptied-q8_0", "acme/repo", "emptied-Q8_0.gguf", "cc")) out := apply(ix, []Family{{Parent: "emptied", Proposals: []Proposal{{Variant: "emptied-q8_0"}}}}) - Expect(strings.Join(out[:4], "\n")).To(Equal("- name: emptied\n variants:\n - model: emptied-q8_0\n url: u")) + Expect(strings.Join(out[3:7], "\n")).To(Equal(" model:\n variants:\n - model: emptied-q8_0\n url: u")) Expect(strings.Count(strings.Join(out, "\n"), "variants:")).To(Equal(1)) }) @@ -81,7 +90,7 @@ var _ = Describe("ApplyFamilies", func() { entryYAML("phi-2-chat:Q8_0", "acme/repo", "phi-2-chat-Q8_0.gguf", "bb"), ) out := apply(ix, []Family{{Parent: "phi-2-chat", Proposals: []Proposal{{Variant: "phi-2-chat:Q8_0"}}}}) - Expect(out[2]).To(Equal(` - model: "phi-2-chat:Q8_0"`)) + Expect(out[ix.Entries[0].EndLine+1]).To(Equal(` - model: "phi-2-chat:Q8_0"`)) // The result has to still be a gallery, and the reference has to // resolve to the entry it names. @@ -119,35 +128,36 @@ var _ = Describe("ApplyFamilies", func() { }) var _ = Describe("ParseIndex", func() { - It("records the anchor an entry defines and the anchor an entry merges", func() { - ix := indexOf(`- &anc - name: anchored - url: u -`, `- !!merge <<: *anc - name: child -`) - Expect(ix.Entries[0].AnchorName).To(Equal("anc")) - Expect(ix.Entries[1].MergesFrom).To(Equal("anc")) - Expect(ix.MergeChildren("anc")).To(HaveLen(1)) - }) - - It("carries merged values into the child, so an inherited variants key is visible", func() { - ix := indexOf(`- &anc - name: anchored - url: u - variants: - - model: something -`, `- !!merge <<: *anc - name: child -`) - Expect(ix.Entries[1].HasVariants()).To(BeTrue()) - }) - It("refuses a list item that decodes to nothing", func() { // Every line number the editor works from comes from pairing decoded // entries with top level list items. If those two views can disagree, // the editor writes into the wrong entry, so the parse refuses instead. - _, err := ParseIndex("- name: one\n url: u\n-\n") + _, err := ParseIndex("- kind: model\n id: one\n-\n") Expect(err).To(MatchError(ContainSubstring("empty"))) }) }) + +var _ = Describe("the checked-in model library", func() { + It("parses, and a proposal edit keeps it parsing with the same entries", func() { + ix, err := LoadIndex("../../../gallery/models/index.yaml") + Expect(err).ToNot(HaveOccurred()) + Expect(ix.Entries).ToNot(BeEmpty()) + for _, e := range ix.Entries { + Expect(e.Name).ToNot(BeEmpty()) + } + + // Add one variant to the first and the last entry: the first has a + // block, the last may have none, and both must stay valid. + first, last := ix.Entries[0], ix.Entries[len(ix.Entries)-1] + out, err := ApplyFamilies(ix, []Family{ + {Parent: first.Name, Proposals: []Proposal{{Variant: "x-added-one"}}}, + {Parent: last.Name, Proposals: []Proposal{{Variant: "x-added-two"}}}, + }) + Expect(err).ToNot(HaveOccurred()) + reparsed, err := ParseIndex(strings.Join(out, "\n")) + Expect(err).ToNot(HaveOccurred()) + Expect(reparsed.Entries).To(HaveLen(len(ix.Entries))) + Expect(reparsed.Entries[0].Variants).To(ContainElement(VariantRef{Model: "x-added-one"})) + Expect(reparsed.Entries[len(ix.Entries)-1].Variants).To(ContainElement(VariantRef{Model: "x-added-two"})) + }) +}) diff --git a/.github/ci/variantproposals/index.go b/.github/ci/variantproposals/index.go index 6cce3699d..ed2f2c3b9 100644 --- a/.github/ci/variantproposals/index.go +++ b/.github/ci/variantproposals/index.go @@ -24,46 +24,50 @@ type VariantRef struct { Model string `yaml:"model"` } -// GalleryEntry is one gallery entry, carrying both the semantics the heuristics need -// and the text range the editor needs. +// GalleryEntry is one model library entry, carrying both the semantics the +// heuristics need and the text range the editor needs. The name is the id of +// the entry, which is the name the gallery knows the model by. // // The two views are kept together deliberately. The editor must not round-trip -// the index through a YAML marshaller: the gallery is 40,000 lines and a +// the index through a YAML marshaller: the library is 40,000 lines and a // reflowed diff cannot be reviewed, which defeats the entire point of a job // whose output is a human decision. type GalleryEntry struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - ConfigFile map[string]any `yaml:"config_file"` - Overrides map[string]any `yaml:"overrides"` - Files []File `yaml:"files"` - Variants []VariantRef `yaml:"variants"` + Name string + URL string + ConfigFile map[string]any + Overrides map[string]any + Files []File + Variants []VariantRef - // Index is the entry's position in gallery order. - Index int `yaml:"-"` + // Index is the entry's position in library order. + Index int // StartLine and EndLine bound the entry's lines, zero based and half open. - StartLine int `yaml:"-"` - EndLine int `yaml:"-"` - // AnchorName is set when the entry defines a YAML anchor. Adding a variants - // key to such an entry is inherited by everything that merges it, which is - // why proposals involving anchors get special treatment. - AnchorName string `yaml:"-"` - // MergesFrom is the anchor this entry pulls in with "!!merge <<:". - MergesFrom string `yaml:"-"` + StartLine int + EndLine int } -// Index is a parsed gallery index: entries plus the exact lines they came from. +// libraryEntry is the subset of a model library entry the proposer decodes. +// Everything but the id lives under model: in the library. +type libraryEntry struct { + Kind string `yaml:"kind"` + ID string `yaml:"id"` + Model struct { + URL string `yaml:"url"` + ConfigFile map[string]any `yaml:"configFile"` + Overrides map[string]any `yaml:"overrides"` + Files []File `yaml:"files"` + Variants []VariantRef `yaml:"variants"` + } `yaml:"model"` +} + +// Index is a parsed model library: entries plus the exact lines they came from. type Index struct { Lines []string Entries []*GalleryEntry } -var ( - anchorStart = regexp.MustCompile(`^- &(\S+)`) - mergeStart = regexp.MustCompile(`^- !!merge <<: \*(\S+)`) -) - -// LoadIndex reads and parses a gallery index file. +// LoadIndex reads and parses a model library file. func LoadIndex(path string) (*Index, error) { data, err := os.ReadFile(path) if err != nil { @@ -72,55 +76,51 @@ func LoadIndex(path string) (*Index, error) { return ParseIndex(string(data)) } -// ParseIndex builds an Index from the raw text of a gallery index. +// ParseIndex builds an Index from the raw text of a model library. // // The YAML decode and the textual scan are cross checked against each other: if // they disagree on how many entries there are, every line number the editor // would use is suspect, so the run fails rather than editing the wrong entry. func ParseIndex(text string) (*Index, error) { - var entries []*GalleryEntry - if err := yaml.Unmarshal([]byte(text), &entries); err != nil { - return nil, fmt.Errorf("decoding gallery index: %w", err) + var decoded []*libraryEntry + if err := yaml.Unmarshal([]byte(text), &decoded); err != nil { + return nil, fmt.Errorf("decoding model library: %w", err) } lines, starts := galleryedit.Scan(text) - if len(starts) != len(entries) { - return nil, fmt.Errorf("gallery index has %d decoded entries but %d top level list items; refusing to edit by line number", len(entries), len(starts)) + if len(starts) != len(decoded) { + return nil, fmt.Errorf("model library has %d decoded entries but %d top level list items; refusing to edit by line number", len(decoded), len(starts)) } - for i, e := range entries { - if e == nil { - return nil, fmt.Errorf("gallery index list item %d is empty; refusing to edit by line number", i) + entries := make([]*GalleryEntry, 0, len(decoded)) + for i, d := range decoded { + if d == nil { + return nil, fmt.Errorf("model library list item %d is empty; refusing to edit by line number", i) + } + if d.Kind != "model" { + return nil, fmt.Errorf("model library list item %d is of kind %q; this job only reads model entries", i, d.Kind) + } + e := &GalleryEntry{ + Name: d.ID, + URL: d.Model.URL, + ConfigFile: d.Model.ConfigFile, + Overrides: d.Model.Overrides, + Files: d.Model.Files, + Variants: d.Model.Variants, + Index: i, + StartLine: starts[i], } - e.Index = i - e.StartLine = starts[i] if i+1 < len(starts) { e.EndLine = starts[i+1] } else { e.EndLine = len(lines) } - if m := anchorStart.FindStringSubmatch(lines[e.StartLine]); m != nil { - e.AnchorName = m[1] - } - if m := mergeStart.FindStringSubmatch(lines[e.StartLine]); m != nil { - e.MergesFrom = m[1] - } + entries = append(entries, e) } return &Index{Lines: lines, Entries: entries}, nil } -// MergeChildren lists the entries that pull in the given anchor. -func (ix *Index) MergeChildren(anchor string) []*GalleryEntry { - var out []*GalleryEntry - for _, e := range ix.Entries { - if e.MergesFrom == anchor { - out = append(out, e) - } - } - return out -} - // ByName indexes entries by lowercased name. A name appearing twice keeps the // first occurrence, matching the gallery's own first-match-wins resolution, and // the duplicates are returned so the caller can refuse to touch them: a diff --git a/.github/ci/variantproposals/main.go b/.github/ci/variantproposals/main.go index 893de7f55..cb6af3ab6 100644 --- a/.github/ci/variantproposals/main.go +++ b/.github/ci/variantproposals/main.go @@ -16,10 +16,10 @@ import ( ) func main() { - index := flag.String("index", "gallery/index.yaml", "path to the gallery index") + index := flag.String("index", "gallery/models/index.yaml", "path to the model library") ledger := flag.String("ledger", "gallery/variant-exclusions.yaml", "path to the rejection ledger") bodyOut := flag.String("body-out", "", "write the pull request body here") - apply := flag.Bool("apply", false, "write the proposed groupings back into the index") + apply := flag.Bool("apply", false, "write the proposed groupings back into the library") flag.Parse() if err := run(*index, *ledger, *bodyOut, *apply); err != nil { diff --git a/.github/ci/variantproposals/propose.go b/.github/ci/variantproposals/propose.go index 42105286b..90128850e 100644 --- a/.github/ci/variantproposals/propose.go +++ b/.github/ci/variantproposals/propose.go @@ -407,25 +407,6 @@ func buildFamily(ix *Index, members []int, evidenceFor map[[2]int]Evidence, clai return nil, &Refusal{Members: names, Reason: fmt.Sprintf("the natural parent %q is already proposed as a variant of %q; proposing it as a parent would nest variants", parentEntry.Name, owner)} } - // Adding a variants key to an anchor is inherited by every entry that - // merges it, silently grouping models nobody proposed. Handling that means - // editing each merging child too, which is a larger change than this job - // should make unsupervised, so it refuses and hands the reviewer the list. - if parentEntry.AnchorName != "" { - children := ix.MergeChildren(parentEntry.AnchorName) - if len(children) > 0 { - childNames := make([]string, 0, len(children)) - for _, c := range children { - childNames = append(childNames, c.Name) - } - return nil, &Refusal{ - Members: names, - Reason: fmt.Sprintf("the parent %q defines YAML anchor &%s, and a variants key added there is inherited by the %d entries that merge it (%s). Grouping this family by hand also means adding an explicit `variants: []` to each of those entries", - parentEntry.Name, parentEntry.AnchorName, len(children), strings.Join(childNames, ", ")), - } - } - } - existing := map[string]struct{}{} for _, v := range parentEntry.Variants { existing[strings.ToLower(v.Model)] = struct{}{} @@ -445,7 +426,7 @@ func buildFamily(ix *Index, members []int, evidenceFor map[[2]int]Evidence, clai return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q already offers variants of its own, so it cannot itself be a variant target", target.Name)} } if !target.Installable() { - return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q has no url, config_file, overrides or files, so it is not independently installable", target.Name)} + return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q has no url, configFile, overrides or files, so it is not independently installable", target.Name)} } if owner, claimed := claimedBy[lower]; claimed && owner != parentName { return nil, &Refusal{Members: names, Reason: fmt.Sprintf("%q is already a variant of %q; a target claimed by two parents is not something the gallery resolves predictably", target.Name, owner)} diff --git a/.github/ci/variantproposals/propose_test.go b/.github/ci/variantproposals/propose_test.go index 2380007f6..a272d9b41 100644 --- a/.github/ci/variantproposals/propose_test.go +++ b/.github/ci/variantproposals/propose_test.go @@ -8,20 +8,23 @@ import ( . "github.com/onsi/gomega" ) -// entryYAML writes one gallery entry with a single weight file, which is the +// entryYAML writes one library entry with a single weight file, which is the // shape almost every real entry has. Specs that need something else write the // YAML out by hand. func entryYAML(name, repo, filename, sha string) string { - return fmt.Sprintf(`- name: %s - url: github:mudler/LocalAI/gallery/virtual.yaml@master - overrides: - parameters: - model: %s - files: - - filename: %s - uri: huggingface://%s/%s - sha256: %s -`, name, filename, filename, repo, filename, sha) + return fmt.Sprintf(`- kind: model + id: %s + name: %s + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + overrides: + parameters: + model: %s + files: + - filename: %s + uri: huggingface://%s/%s + sha256: %s +`, name, name, filename, filename, repo, filename, sha) } func indexOf(entries ...string) *Index { @@ -103,24 +106,30 @@ var _ = Describe("Propose", func() { // Both entries ship the same text encoder. That is a packaging // convention, not evidence of shared weights: this is how an // earlier sweep linked four wan-2.1 entries to each other. - ix := indexOf(`- name: wan-2.1-t2v - url: u - files: - - filename: wan-2.1-t2v-Q4_K_M.gguf - uri: huggingface://acme/wan/wan-2.1-t2v-Q4_K_M.gguf - sha256: aa - - filename: umt5-xxl-encoder-Q8_0.gguf - uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf - sha256: cc -`, `- name: z-image-turbo - url: u - files: - - filename: z-image-turbo-Q4_K_M.gguf - uri: huggingface://acme/wan/z-image-turbo-Q4_K_M.gguf - sha256: bb - - filename: umt5-xxl-encoder-Q8_0.gguf - uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf - sha256: cc + ix := indexOf(`- kind: model + id: wan-2.1-t2v + name: wan-2.1-t2v + model: + url: u + files: + - filename: wan-2.1-t2v-Q4_K_M.gguf + uri: huggingface://acme/wan/wan-2.1-t2v-Q4_K_M.gguf + sha256: aa + - filename: umt5-xxl-encoder-Q8_0.gguf + uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf + sha256: cc +`, `- kind: model + id: z-image-turbo + name: z-image-turbo + model: + url: u + files: + - filename: z-image-turbo-Q4_K_M.gguf + uri: huggingface://acme/wan/z-image-turbo-Q4_K_M.gguf + sha256: bb + - filename: umt5-xxl-encoder-Q8_0.gguf + uri: huggingface://acme/wan/umt5-xxl-encoder-Q8_0.gguf + sha256: cc `) r := Propose(ix, nil) Expect(familyNames(r)).To(BeEmpty()) @@ -162,16 +171,17 @@ var _ = Describe("Propose", func() { It("does not group entries with a byte-identical install payload", func() { // whisper-1 exists so OpenAI-compatible clients can send that name. // Folding it under whisper-base would hide the name they send. - payload := ` url: github:mudler/LocalAI/gallery/whisper-base.yaml@master - overrides: - parameters: - model: ggml-whisper-base.bin - files: - - filename: ggml-whisper-base.bin - uri: huggingface://ggerganov/whisper.cpp/ggml-base.bin - sha256: aa + payload := ` model: + url: github:mudler/LocalAI/gallery/whisper-base.yaml@master + overrides: + parameters: + model: ggml-whisper-base.bin + files: + - filename: ggml-whisper-base.bin + uri: huggingface://ggerganov/whisper.cpp/ggml-base.bin + sha256: aa ` - ix := indexOf("- name: whisper-base\n"+payload, "- name: whisper-1\n"+payload) + ix := indexOf("- kind: model\n id: whisper-base\n"+payload, "- kind: model\n id: whisper-1\n"+payload) r := Propose(ix, nil) Expect(familyNames(r)).To(BeEmpty()) Expect(r.AliasSkipped).To(HaveLen(1)) @@ -256,17 +266,20 @@ var _ = Describe("Propose", func() { It("refuses to nest: a target that already offers variants of its own", func() { ix := indexOf( entryYAML("nest-model", "acme/repo", "nest-model-Q4_K_M.gguf", "aa"), - `- name: nest-model-q8_0 - url: u - variants: - - model: nest-model-q8_0-mtp - overrides: - parameters: - model: nest-model-Q8_0.gguf - files: - - filename: nest-model-Q8_0.gguf - uri: huggingface://acme/repo/nest-model-Q8_0.gguf - sha256: bb + `- kind: model + id: nest-model-q8_0 + name: nest-model-q8_0 + model: + url: u + variants: + - model: nest-model-q8_0-mtp + overrides: + parameters: + model: nest-model-Q8_0.gguf + files: + - filename: nest-model-Q8_0.gguf + uri: huggingface://acme/repo/nest-model-Q8_0.gguf + sha256: bb `, entryYAML("nest-model-q8_0-mtp", "other/repo", "nest-model-mtp.gguf", "cc"), ) @@ -277,17 +290,20 @@ var _ = Describe("Propose", func() { It("refuses to nest: a parent that is already somebody else's variant", func() { ix := indexOf( - `- name: outer - url: u - variants: - - model: middle - overrides: - parameters: - model: outer-Q4_K_M.gguf - files: - - filename: outer-Q4_K_M.gguf - uri: huggingface://acme/repo/outer-Q4_K_M.gguf - sha256: aa + `- kind: model + id: outer + name: outer + model: + url: u + variants: + - model: middle + overrides: + parameters: + model: outer-Q4_K_M.gguf + files: + - filename: outer-Q4_K_M.gguf + uri: huggingface://acme/repo/outer-Q4_K_M.gguf + sha256: aa `, entryYAML("middle", "acme/other", "middle-Q4_K_M.gguf", "bb"), entryYAML("middle-q8_0", "acme/other", "middle-Q8_0.gguf", "cc"), @@ -299,17 +315,20 @@ var _ = Describe("Propose", func() { It("refuses to let two parents claim one target", func() { ix := indexOf( - `- name: claimant - url: u - variants: - - model: contested-q8_0 - overrides: - parameters: - model: claimant-Q4_K_M.gguf - files: - - filename: claimant-Q4_K_M.gguf - uri: huggingface://acme/repo/claimant-Q4_K_M.gguf - sha256: aa + `- kind: model + id: claimant + name: claimant + model: + url: u + variants: + - model: contested-q8_0 + overrides: + parameters: + model: claimant-Q4_K_M.gguf + files: + - filename: claimant-Q4_K_M.gguf + uri: huggingface://acme/repo/claimant-Q4_K_M.gguf + sha256: aa `, entryYAML("contested", "acme/other", "contested-Q4_K_M.gguf", "bb"), entryYAML("contested-q8_0", "acme/other", "contested-Q8_0.gguf", "cc"), @@ -322,46 +341,13 @@ var _ = Describe("Propose", func() { It("refuses a target that is not independently installable", func() { ix := indexOf( entryYAML("stub-model", "acme/repo", "stub-model-Q4_K_M.gguf", "aa"), - "- name: stub-model-q8_0\n description: a stanza nobody finished\n", + "- kind: model\n id: stub-model-q8_0\n name: stub-model-q8_0\n description: a stanza nobody finished\n model: {}\n", ) r := Propose(ix, nil) Expect(familyNames(r)).To(BeEmpty()) Expect(refusalReasons(r)).To(ContainSubstring("not independently installable")) }) - It("refuses a family whose parent defines a merge anchor, naming the entries that would inherit", func() { - ix := indexOf( - `- &anchored - name: anchored-model - url: u - overrides: - parameters: - model: anchored-Q4_K_M.gguf - files: - - filename: anchored-Q4_K_M.gguf - uri: huggingface://acme/repo/anchored-Q4_K_M.gguf - sha256: aa -`, - `- !!merge <<: *anchored - name: anchored-child - variants: [] - overrides: - parameters: - model: unrelated-child-Q4_K_M.gguf - files: - - filename: unrelated-child-Q4_K_M.gguf - uri: huggingface://other/repo/unrelated-child-Q4_K_M.gguf - sha256: cc -`, - entryYAML("anchored-model-q8_0", "acme/repo", "anchored-Q8_0.gguf", "bb"), - ) - r := Propose(ix, nil) - Expect(familyNames(r)).To(BeEmpty()) - Expect(refusalReasons(r)).To(ContainSubstring("defines YAML anchor &anchored")) - Expect(refusalReasons(r)).To(ContainSubstring("anchored-child")) - Expect(refusalReasons(r)).To(ContainSubstring("variants: []")) - }) - It("refuses an entry whose name is not unique in the gallery", func() { ix := indexOf( entryYAML("twin", "acme/repo", "twin-Q4_K_M.gguf", "aa"), @@ -375,17 +361,20 @@ var _ = Describe("Propose", func() { It("says nothing about a pair that is already grouped", func() { ix := indexOf( - `- name: settled - url: u - variants: - - model: settled-q8_0 - overrides: - parameters: - model: settled-Q4_K_M.gguf - files: - - filename: settled-Q4_K_M.gguf - uri: huggingface://acme/repo/settled-Q4_K_M.gguf - sha256: aa + `- kind: model + id: settled + name: settled + model: + url: u + variants: + - model: settled-q8_0 + overrides: + parameters: + model: settled-Q4_K_M.gguf + files: + - filename: settled-Q4_K_M.gguf + uri: huggingface://acme/repo/settled-Q4_K_M.gguf + sha256: aa `, entryYAML("settled-q8_0", "acme/repo", "settled-Q8_0.gguf", "bb"), ) @@ -397,17 +386,20 @@ var _ = Describe("Propose", func() { It("adds only the missing members to a family that already exists", func() { ix := indexOf( - `- name: partial - url: u - variants: - - model: partial-q8_0 - overrides: - parameters: - model: partial-Q4_K_M.gguf - files: - - filename: partial-Q4_K_M.gguf - uri: huggingface://acme/repo/partial-Q4_K_M.gguf - sha256: aa + `- kind: model + id: partial + name: partial + model: + url: u + variants: + - model: partial-q8_0 + overrides: + parameters: + model: partial-Q4_K_M.gguf + files: + - filename: partial-Q4_K_M.gguf + uri: huggingface://acme/repo/partial-Q4_K_M.gguf + sha256: aa `, entryYAML("partial-q8_0", "acme/repo", "partial-Q8_0.gguf", "bb"), entryYAML("partial-f16", "acme/repo", "partial-f16.gguf", "cc"), diff --git a/.github/gallery-agent/gallery.go b/.github/gallery-agent/gallery.go index 445fdc789..bdcb96a95 100644 --- a/.github/gallery-agent/gallery.go +++ b/.github/gallery-agent/gallery.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strings" "github.com/mudler/LocalAI/core/gallery/importers" @@ -155,59 +154,32 @@ func generateYAMLEntry(model ProcessedModel, quantization string) string { ) } -// generateYAMLForModels generates YAML entries for selected models and appends to index.yaml +// generateYAMLForModels generates library entries for the selected models and +// appends them to the model library, gallery/models/index.yaml. +// +// The entry text is still built in the gallery shape, because that is what the +// importer output maps onto. appendLibraryEntries converts each entry to the +// library shape before anything is written. func generateYAMLForModels(ctx context.Context, models []ProcessedModel, quantization string) error { - - // Generate YAML entries for each model var yamlEntries []string for _, model := range models { fmt.Printf("Generating YAML entry for model: %s\n", model.ModelID) - - // Generate YAML entry - yamlEntry := generateYAMLEntry(model, quantization) - yamlEntries = append(yamlEntries, yamlEntry) + yamlEntries = append(yamlEntries, generateYAMLEntry(model, quantization)) } - // Prepend to index.yaml (write at the top) - if len(yamlEntries) > 0 { - indexPath := getGalleryIndexPath() - fmt.Printf("Prepending YAML entries to %s...\n", indexPath) - - // Read current content - content, err := os.ReadFile(indexPath) - if err != nil { - return fmt.Errorf("failed to read %s: %w", indexPath, err) - } - - existingContent := string(content) - yamlBlock := strings.Join(yamlEntries, "\n") - - // Check if file starts with "---" - var newContent string - if strings.HasPrefix(existingContent, "---\n") { - // File starts with "---", prepend new entries after it - restOfContent := strings.TrimPrefix(existingContent, "---\n") - // Ensure proper spacing: "---\n" + new entries + "\n" + rest of content - newContent = "---\n" + yamlBlock + "\n" + restOfContent - } else if strings.HasPrefix(existingContent, "---") { - // File starts with "---" but no newline after - restOfContent := strings.TrimPrefix(existingContent, "---") - newContent = "---\n" + yamlBlock + "\n" + strings.TrimPrefix(restOfContent, "\n") - } else { - // No "---" at start, prepend new entries at the very beginning - // Trim leading whitespace from existing content - existingContent = strings.TrimLeft(existingContent, " \t\n\r") - newContent = yamlBlock + "\n" + existingContent - } - - // Write back to file - err = os.WriteFile(indexPath, []byte(newContent), 0644) - if err != nil { - return fmt.Errorf("failed to write %s: %w", indexPath, err) - } - - fmt.Printf("Successfully prepended %d models to %s\n", len(yamlEntries), indexPath) + if len(yamlEntries) == 0 { + return nil } + indexPath := getGalleryIndexPath() + fmt.Printf("Appending library entries to %s...\n", indexPath) + added, skipped, err := appendLibraryEntries(indexPath, yamlEntries) + if err != nil { + return err + } + for _, id := range skipped { + fmt.Printf("Skipped %s: the library already has an entry with this id\n", id) + } + fmt.Printf("Successfully appended %d models to %s\n", len(added), indexPath) return nil } diff --git a/.github/gallery-agent/helpers.go b/.github/gallery-agent/helpers.go index 5f569e6d9..e57a7a822 100644 --- a/.github/gallery-agent/helpers.go +++ b/.github/gallery-agent/helpers.go @@ -13,14 +13,19 @@ import ( "sigs.k8s.io/yaml" ) +// A library entry has the same top-level name and urls keys as a gallery +// entry, so galleryModel below reads both. + var galleryIndexPath = os.Getenv("GALLERY_INDEX_PATH") -// getGalleryIndexPath returns the gallery index file path, with a default fallback +// getGalleryIndexPath returns the model library path, with a default fallback. +// The library, gallery/models/index.yaml, is the source of the models; the +// gallery/index.yaml copy is generated from it and is never written here. func getGalleryIndexPath() string { if galleryIndexPath != "" { return galleryIndexPath } - return "gallery/index.yaml" + return "gallery/models/index.yaml" } type galleryModel struct { @@ -28,7 +33,7 @@ type galleryModel struct { Urls []string `yaml:"urls"` } -// loadGalleryURLSet parses gallery/index.yaml once and returns the set of +// loadGalleryURLSet parses the model library once and returns the set of // HuggingFace model URLs already present in the gallery. func loadGalleryURLSet() (map[string]struct{}, error) { indexPath := getGalleryIndexPath() @@ -142,7 +147,7 @@ func resolveReadme(client *hfapi.Client, modelID string, hfTags []string) (strin } // extractDescription turns a raw HuggingFace README into a concise plain-text -// description suitable for embedding in gallery/index.yaml: strips YAML +// description suitable for embedding in a gallery entry: strips YAML // frontmatter, HTML tags/comments, markdown images, link URLs (keeping the // link text), markdown tables, and then truncates at a paragraph boundary // around ~1200 characters. Raw README should still be used for icon diff --git a/.github/gallery-agent/library.go b/.github/gallery-agent/library.go new file mode 100644 index 000000000..67c74e9a7 --- /dev/null +++ b/.github/gallery-agent/library.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/mudler/LocalAI/core/gallery" + "gopkg.in/yaml.v3" +) + +// appendLibraryEntries converts gallery-shaped entry texts to model library +// entries and appends them to the library file at path. It returns the ids it +// added and the ids it skipped because the library already has them. +// +// The entries are appended as text, after the existing content, so the diff of +// the library is only the new entries. The result is decoded again with the +// reader the host uses before it is written, so a malformed entry fails the +// run instead of reaching a pull request. +func appendLibraryEntries(path string, galleryEntries []string) (added, skipped []string, err error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("failed to read %s: %w", path, err) + } + current, err := gallery.DecodeModelIndex(content) + if err != nil { + return nil, nil, fmt.Errorf("failed to decode %s as a model library: %w", path, err) + } + ids := make(map[string]struct{}, len(current.Entries)) + for _, e := range current.Entries { + ids[e.ID] = struct{}{} + } + + var entries []gallery.ModelLibraryEntry + for _, text := range galleryEntries { + var models []gallery.GalleryModel + if err := yaml.Unmarshal([]byte(text), &models); err != nil { + return nil, nil, fmt.Errorf("generated entry does not decode: %w", err) + } + for _, m := range models { + entry, err := gallery.ModelLibraryEntryFromGallery(m) + if err != nil { + return nil, nil, fmt.Errorf("converting %q to a library entry: %w", m.Name, err) + } + if _, dup := ids[entry.ID]; dup { + skipped = append(skipped, entry.ID) + continue + } + ids[entry.ID] = struct{}{} + entries = append(entries, entry) + added = append(added, entry.ID) + } + } + if len(entries) == 0 { + return nil, skipped, nil + } + + encoded, err := gallery.EncodeModelIndex(entries) + if err != nil { + return nil, nil, fmt.Errorf("encoding library entries: %w", err) + } + + out := string(content) + if !strings.HasSuffix(out, "\n") { + out += "\n" + } + out += string(encoded) + if _, err := gallery.DecodeModelIndex([]byte(out)); err != nil { + return nil, nil, fmt.Errorf("the library with the new entries does not decode: %w", err) + } + if err := os.WriteFile(path, []byte(out), 0o644); err != nil { + return nil, nil, fmt.Errorf("failed to write %s: %w", path, err) + } + return added, skipped, nil +} diff --git a/.github/gallery-agent/library_test.go b/.github/gallery-agent/library_test.go new file mode 100644 index 000000000..7161a2920 --- /dev/null +++ b/.github/gallery-agent/library_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mudler/LocalAI/core/gallery" +) + +const libraryFixture = `# header comment +- kind: model + id: existing-model + name: existing-model + version: 0.0.0-gallery.000000000000 + description: | + An existing model. + urls: + - https://huggingface.co/acme/existing-GGUF + channel: gallery + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master +` + +// agentEntry has the shape generateYAMLEntry writes. +const agentEntry = `- name: "new-model" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/acme/new-model-GGUF + description: | + A new model. + license: "apache-2.0" + tags: + - llm + - gguf + overrides: + parameters: + model: new-model-Q4_K_M.gguf + files: + - filename: new-model-Q4_K_M.gguf + sha256: abc + uri: huggingface://acme/new-model-GGUF/new-model-Q4_K_M.gguf +` + +func writeFixture(t *testing.T) string { + t.Helper() + p := filepath.Join(t.TempDir(), "index.yaml") + if err := os.WriteFile(p, []byte(libraryFixture), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func TestAppendLibraryEntriesWritesTheLibraryShape(t *testing.T) { + p := writeFixture(t) + added, skipped, err := appendLibraryEntries(p, []string{agentEntry}) + if err != nil { + t.Fatal(err) + } + if len(added) != 1 || added[0] != "new-model" || len(skipped) != 0 { + t.Fatalf("added %v skipped %v", added, skipped) + } + + got, _ := os.ReadFile(p) + if !strings.HasPrefix(string(got), libraryFixture) { + t.Fatal("existing content was changed; entries must only be appended") + } + res, err := gallery.DecodeModelIndex(got) + if err != nil { + t.Fatal(err) + } + if len(res.Entries) != 2 { + t.Fatalf("want 2 entries, got %d", len(res.Entries)) + } + e := res.Entries[1] + if e.ID != "new-model" || e.Kind != "model" { + t.Fatalf("unexpected entry %+v", e.Envelope) + } + if e.Model.URL == "" || len(e.Model.Files) != 1 || e.Model.Overrides == nil { + t.Fatalf("model body not filled: %+v", e.Model) + } + if len(e.URLs) != 1 || e.URLs[0] != "https://huggingface.co/acme/new-model-GGUF" { + t.Fatalf("urls: %v", e.URLs) + } +} + +func TestAppendLibraryEntriesSkipsAnExistingID(t *testing.T) { + p := writeFixture(t) + dup := strings.Replace(agentEntry, `"new-model"`, `"existing-model"`, 1) + added, skipped, err := appendLibraryEntries(p, []string{dup}) + if err != nil { + t.Fatal(err) + } + if len(added) != 0 || len(skipped) != 1 { + t.Fatalf("added %v skipped %v", added, skipped) + } + got, _ := os.ReadFile(p) + if string(got) != libraryFixture { + t.Fatal("file changed although nothing was added") + } +} + +func TestDuplicateCheckReadsTheLibrary(t *testing.T) { + p := writeFixture(t) + if _, _, err := appendLibraryEntries(p, []string{agentEntry}); err != nil { + t.Fatal(err) + } + t.Setenv("GALLERY_INDEX_PATH", p) + old := galleryIndexPath + galleryIndexPath = p + defer func() { galleryIndexPath = old }() + + set, err := loadGalleryURLSet() + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"acme/existing-GGUF", "acme/new-model-GGUF"} { + if !modelAlreadyInGallery(set, id) { + t.Fatalf("%s should count as already in the library", id) + } + } +} diff --git a/.github/gallery-agent/testing.go b/.github/gallery-agent/testing.go index 289f74f3f..6a1e1a5e1 100644 --- a/.github/gallery-agent/testing.go +++ b/.github/gallery-agent/testing.go @@ -8,7 +8,7 @@ import ( "time" ) -// runSyntheticMode generates synthetic test data and appends it to the gallery +// runSyntheticMode generates synthetic test data and appends it to the model library func runSyntheticMode() error { generator := NewSyntheticDataGenerator() @@ -23,7 +23,7 @@ func runSyntheticMode() error { fmt.Printf("Generated synthetic model: %s\n", model.ModelID) } - // Generate YAML entries and append to gallery/index.yaml + // Generate YAML entries and append to the model library fmt.Println("Generating YAML entries for synthetic models...") err := generateYAMLForModels(context.Background(), models, "Q4_K_M") if err != nil { diff --git a/.github/workflows/checksum_checker.yaml b/.github/workflows/checksum_checker.yaml index 3652c65bb..ece1b7c84 100644 --- a/.github/workflows/checksum_checker.yaml +++ b/.github/workflows/checksum_checker.yaml @@ -16,6 +16,19 @@ jobs: sudo apt-get update sudo apt-get install -y pip wget pip install huggingface_hub + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + - name: Generate the gRPC stubs the Go packages import + # modelcatalog imports them and they are not in the tree. Same pinned + # tools as the model library workflow. + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip + sudo unzip -j -d /usr/local/bin protoc.zip bin/protoc + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go - name: 'Setup yq' uses: dcarbone/install-yq-action@v1.3.1 with: @@ -28,14 +41,19 @@ jobs: export HF_HOME=/hf_cache sudo mkdir /hf_cache sudo chmod 777 /hf_cache - bash .github/checksum_checker.sh gallery/index.yaml + # The model library is the source. The checker edits it. + bash .github/checksum_checker.sh gallery/models/index.yaml + - name: Regenerate the gallery copy from the library + # gallery/index.yaml is generated; the model library check fails the + # PR when it differs from the generated copy. + run: go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml - name: Create Pull Request uses: peter-evans/create-pull-request@v8 with: token: ${{ secrets.UPDATE_BOT_TOKEN }} push-to-fork: ci-forks/LocalAI - commit-message: ':arrow_up: Checksum updates in gallery/index.yaml' + commit-message: ':arrow_up: Checksum updates in the model library' title: 'chore(model-gallery): :arrow_up: update checksum' branch: "update/checksum" - body: Updating checksums in gallery/index.yaml + body: Updating checksums in gallery/models/index.yaml and regenerating gallery/index.yaml signoff: true diff --git a/.github/workflows/gallery-agent.yaml b/.github/workflows/gallery-agent.yaml index dfe9d40fa..b0dbe61c1 100644 --- a/.github/workflows/gallery-agent.yaml +++ b/.github/workflows/gallery-agent.yaml @@ -38,7 +38,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version-file: go.mod - name: Proto Dependencies run: | # Install protoc @@ -131,19 +131,25 @@ jobs: MAX_MODELS: ${{ github.event.inputs.max_models || '1' }} EXTRA_SKIP_URLS: ${{ steps.open_prs.outputs.urls }} run: | - export GALLERY_INDEX_PATH=$PWD/gallery/index.yaml + # The agent reads and appends to the model library, the source. + export GALLERY_INDEX_PATH=$PWD/gallery/models/index.yaml go run ./.github/gallery-agent + - name: Regenerate the gallery copy from the library + # gallery/index.yaml is generated from the library; the model library + # check fails the PR when they disagree. + run: go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml + - name: Check for changes id: check_changes run: | - if git diff --quiet gallery/index.yaml; then + if git diff --quiet gallery/models/index.yaml gallery/index.yaml; then echo "changes=false" >> $GITHUB_OUTPUT - echo "No changes detected in gallery/index.yaml" + echo "No changes detected in the model library" else echo "changes=true" >> $GITHUB_OUTPUT - echo "Changes detected in gallery/index.yaml" - git diff gallery/index.yaml + echo "Changes detected in the model library" + git diff gallery/models/index.yaml gallery/index.yaml fi - name: Read gallery agent summary diff --git a/.github/workflows/gallery_variant_proposals.yaml b/.github/workflows/gallery_variant_proposals.yaml index b7753adc5..58d7cef7c 100644 --- a/.github/workflows/gallery_variant_proposals.yaml +++ b/.github/workflows/gallery_variant_proposals.yaml @@ -19,12 +19,24 @@ jobs: - name: Test the proposer run: go test ./.github/ci/variantproposals/ + # The proposer calls no gallery code, but modelcatalog imports packages + # that need the generated gRPC stubs. Same pinned tools as the model + # library workflow. + - name: Generate the gRPC stubs the Go packages import + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip + sudo unzip -j -d /usr/local/bin protoc.zip bin/protoc + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Propose groupings 🔧 id: propose run: | rm -f /tmp/variant-proposals-body.md go run ./.github/ci/variantproposals \ - -index gallery/index.yaml \ + -index gallery/models/index.yaml \ -ledger gallery/variant-exclusions.yaml \ -body-out /tmp/variant-proposals-body.md \ -apply @@ -39,6 +51,13 @@ jobs: echo "have_proposals=false" >> "$GITHUB_OUTPUT" fi + # The proposer edits the model library, the source. The gallery copy is + # generated from it, and the model library check fails a PR where they + # disagree. Only regenerate when the library changed. + - name: Regenerate the gallery copy from the library + if: steps.propose.outputs.have_proposals == 'true' + run: go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml + # No body file means the proposer found nothing. Opening an empty pull # request every run is how a proposal job gets muted by its reviewers. - name: Create Pull Request