mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-04-02 14:16:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore(ci/agent): support quantization

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-12-04 22:56:35 +01:00
parent 963796ff51
commit 3e9ed48432
3 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.github/gallery-agent/gallery.go vendored
View File

@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
@@ -29,9 +30,8 @@ func formatTextContent(text string) string {
}
// generateYAMLEntry generates a YAML entry for a model using the specified anchor
func generateYAMLEntry(model ProcessedModel, familyAnchor string) string {
modelConfig, err := importers.DiscoverModelConfig("https://huggingface.co/"+model.ModelID, nil)
func generateYAMLEntry(model ProcessedModel, familyAnchor string, quantization string) string {
modelConfig, err := importers.DiscoverModelConfig("https://huggingface.co/"+model.ModelID, json.RawMessage(`{ "quantization": "`+quantization+`"}`))
if err != nil {
panic(err)
}
@@ -120,7 +120,7 @@ func extractModelFamilies() ([]ModelFamily, error) {
}
// generateYAMLForModels generates YAML entries for selected models and appends to index.yaml
func generateYAMLForModels(ctx context.Context, models []ProcessedModel) error {
func generateYAMLForModels(ctx context.Context, models []ProcessedModel, quantization string) error {
// Extract available model families
families, err := extractModelFamilies()
if err != nil {
@@ -151,7 +151,7 @@ func generateYAMLForModels(ctx context.Context, models []ProcessedModel) error {
fmt.Printf("Selected family '%s' for model %s\n", familyAnchor, model.ModelID)
// Generate YAML entry
yamlEntry := generateYAMLEntry(model, familyAnchor)
yamlEntry := generateYAMLEntry(model, familyAnchor, quantization)
yamlEntries = append(yamlEntries, yamlEntry)
}

2
.github/gallery-agent/main.go vendored
View File

@@ -154,7 +154,7 @@ func main() {
addedModelURLs = append(addedModelURLs, modelURL)
}
fmt.Println("Generating YAML entries for selected models...")
err = generateYAMLForModels(context.Background(), models)
err = generateYAMLForModels(context.Background(), models, quantization)
if err != nil {
fmt.Fprintf(os.Stderr, "Error generating YAML entries: %v\n", err)
os.Exit(1)

2
.github/gallery-agent/testing.go vendored
View File

@@ -25,7 +25,7 @@ func runSyntheticMode() error {
// Generate YAML entries and append to gallery/index.yaml
fmt.Println("Generating YAML entries for synthetic models...")
err := generateYAMLForModels(context.Background(), models)
err := generateYAMLForModels(context.Background(), models, "Q4_K_M")
if err != nil {
return fmt.Errorf("error generating YAML entries: %w", err)
}