add libx11

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.air.toml

@@ -1,8 +0,0 @@
-# .air.toml
-[build]
-cmd = "make build"
-bin = "./local-ai"
-args_bin = [ "--debug" ]
-include_ext = ["go", "html", "yaml", "toml", "json", "txt", "md"]
-exclude_dir = ["pkg/grpc/proto"]
-delay = 1000

.env

@@ -32,6 +32,15 @@
 # Forces shutdown of the backends if busy (only if LOCALAI_SINGLE_ACTIVE_BACKEND is set)
 # LOCALAI_FORCE_BACKEND_SHUTDOWN=true
 
+## Specify a build type. Available: cublas, openblas, clblas.
+## cuBLAS: This is a GPU-accelerated version of the complete standard BLAS (Basic Linear Algebra Subprograms) library. It's provided by Nvidia and is part of their CUDA toolkit.
+## OpenBLAS: This is an open-source implementation of the BLAS library that aims to provide highly optimized code for various platforms. It includes support for multi-threading and can be compiled to use hardware-specific features for additional performance. OpenBLAS can run on many kinds of hardware, including CPUs from Intel, AMD, and ARM.
+## clBLAS:   This is an open-source implementation of the BLAS library that uses OpenCL, a framework for writing programs that execute across heterogeneous platforms consisting of CPUs, GPUs, and other processors. clBLAS is designed to take advantage of the parallel computing power of GPUs but can also run on any hardware that supports OpenCL. This includes hardware from different vendors like Nvidia, AMD, and Intel.
+# BUILD_TYPE=openblas
+
+## Uncomment and set to true to enable rebuilding from source
+# REBUILD=true
+
 ## Path where to store generated images
 # LOCALAI_IMAGE_PATH=/tmp/generated/images
 

.github/gallery-agent/agent.go

@@ -1,445 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"regexp"
-	"slices"
-	"strings"
-
-	"github.com/ghodss/yaml"
-	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
-	cogito "github.com/mudler/cogito"
-
-	"github.com/mudler/cogito/structures"
-	"github.com/sashabaranov/go-openai/jsonschema"
-)
-
-var (
-	openAIModel      = os.Getenv("OPENAI_MODEL")
-	openAIKey        = os.Getenv("OPENAI_KEY")
-	openAIBaseURL    = os.Getenv("OPENAI_BASE_URL")
-	galleryIndexPath = os.Getenv("GALLERY_INDEX_PATH")
-	//defaultclient
-	llm = cogito.NewOpenAILLM(openAIModel, openAIKey, openAIBaseURL)
-)
-
-// cleanTextContent removes trailing spaces, tabs, and normalizes line endings
-// to prevent YAML linting issues like trailing spaces and multiple empty lines
-func cleanTextContent(text string) string {
-	lines := strings.Split(text, "\n")
-	var cleanedLines []string
-	var prevEmpty bool
-	for _, line := range lines {
-		// Remove all trailing whitespace (spaces, tabs, etc.)
-		trimmed := strings.TrimRight(line, " \t\r")
-		// Avoid multiple consecutive empty lines
-		if trimmed == "" {
-			if !prevEmpty {
-				cleanedLines = append(cleanedLines, "")
-			}
-			prevEmpty = true
-		} else {
-			cleanedLines = append(cleanedLines, trimmed)
-			prevEmpty = false
-		}
-	}
-	// Remove trailing empty lines from the result
-	result := strings.Join(cleanedLines, "\n")
-	return stripThinkingTags(strings.TrimRight(result, "\n"))
-}
-
-type galleryModel struct {
-	Name string   `yaml:"name"`
-	Urls []string `yaml:"urls"`
-}
-
-// isModelExisting checks if a specific model ID exists in the gallery using text search
-func isModelExisting(modelID string) (bool, error) {
-	indexPath := getGalleryIndexPath()
-	content, err := os.ReadFile(indexPath)
-	if err != nil {
-		return false, fmt.Errorf("failed to read %s: %w", indexPath, err)
-	}
-
-	var galleryModels []galleryModel
-
-	err = yaml.Unmarshal(content, &galleryModels)
-	if err != nil {
-		return false, fmt.Errorf("failed to unmarshal %s: %w", indexPath, err)
-	}
-
-	for _, galleryModel := range galleryModels {
-		if slices.Contains(galleryModel.Urls, modelID) {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
-// filterExistingModels removes models that already exist in the gallery
-func filterExistingModels(models []ProcessedModel) ([]ProcessedModel, error) {
-	var filteredModels []ProcessedModel
-	for _, model := range models {
-		exists, err := isModelExisting(model.ModelID)
-		if err != nil {
-			fmt.Printf("Error checking if model %s exists: %v, skipping\n", model.ModelID, err)
-			continue
-		}
-
-		if !exists {
-			filteredModels = append(filteredModels, model)
-		} else {
-			fmt.Printf("Skipping existing model: %s\n", model.ModelID)
-		}
-	}
-
-	fmt.Printf("Filtered out %d existing models, %d new models remaining\n",
-		len(models)-len(filteredModels), len(filteredModels))
-
-	return filteredModels, nil
-}
-
-// getGalleryIndexPath returns the gallery index file path, with a default fallback
-func getGalleryIndexPath() string {
-	if galleryIndexPath != "" {
-		return galleryIndexPath
-	}
-	return "gallery/index.yaml"
-}
-
-func stripThinkingTags(content string) string {
-	// Remove content between <thinking> and </thinking> (including multi-line)
-	content = regexp.MustCompile(`(?s)<thinking>.*?</thinking>`).ReplaceAllString(content, "")
-	// Remove content between <think> and </think> (including multi-line)
-	content = regexp.MustCompile(`(?s)<think>.*?</think>`).ReplaceAllString(content, "")
-	// Clean up any extra whitespace
-	content = strings.TrimSpace(content)
-	return content
-}
-
-func getRealReadme(ctx context.Context, repository string) (string, error) {
-	// Create a conversation fragment
-	fragment := cogito.NewEmptyFragment().
-		AddMessage("user",
-			`Your task is to get a clear description of a large language model from huggingface by using the provided tool. I will share with you a repository that might be quantized, and as such probably not by the original model author. We need to get the real  description of the model, and not the one that might be quantized. You will have to call the tool to get the readme more than once by figuring out from the quantized readme which is the base model readme. This is the repository: `+repository)
-
-	// Execute with tools
-	result, err := cogito.ExecuteTools(llm, fragment,
-		cogito.WithIterations(3),
-		cogito.WithMaxAttempts(3),
-		cogito.WithTools(&HFReadmeTool{client: hfapi.NewClient()}))
-	if err != nil {
-		return "", err
-	}
-
-	result = result.AddMessage("user", "Describe the model in a clear and concise way that can be shared in a model gallery.")
-
-	// Get a response
-	newFragment, err := llm.Ask(ctx, result)
-	if err != nil {
-		return "", err
-	}
-
-	content := newFragment.LastMessage().Content
-	return cleanTextContent(content), nil
-}
-
-func selectMostInterestingModels(ctx context.Context, searchResult *SearchResult) ([]ProcessedModel, error) {
-
-	if len(searchResult.Models) == 1 {
-		return searchResult.Models, nil
-	}
-
-	// Create a conversation fragment
-	fragment := cogito.NewEmptyFragment().
-		AddMessage("user",
-			`Your task is to analyze a list of AI models and select the most interesting ones for a model gallery. You will be given detailed information about multiple models including their metadata, file information, and README content.
-
-Consider the following criteria when selecting models:
-1. Model popularity (download count)
-2. Model recency (last modified date)
-3. Model completeness (has preferred model file, README, etc.)
-4. Model uniqueness (not duplicates or very similar models)
-5. Model quality (based on README content and description)
-6. Model utility (practical applications)
-
-You should select models that would be most valuable for users browsing a model gallery. Prioritize models that are:
-- Well-documented with clear READMEs
-- Recently updated
-- Popular (high download count)
-- Have the preferred quantization format available
-- Offer unique capabilities or are from reputable authors
-
-Return your analysis and selection reasoning.`)
-
-	// Add the search results as context
-	modelsInfo := fmt.Sprintf("Found %d models matching '%s' with quantization preference '%s':\n\n",
-		searchResult.TotalModelsFound, searchResult.SearchTerm, searchResult.Quantization)
-
-	for i, model := range searchResult.Models {
-		modelsInfo += fmt.Sprintf("Model %d:\n", i+1)
-		modelsInfo += fmt.Sprintf("  ID: %s\n", model.ModelID)
-		modelsInfo += fmt.Sprintf("  Author: %s\n", model.Author)
-		modelsInfo += fmt.Sprintf("  Downloads: %d\n", model.Downloads)
-		modelsInfo += fmt.Sprintf("  Last Modified: %s\n", model.LastModified)
-		modelsInfo += fmt.Sprintf("  Files: %d files\n", len(model.Files))
-
-		if model.PreferredModelFile != nil {
-			modelsInfo += fmt.Sprintf("  Preferred Model File: %s (%d bytes)\n",
-				model.PreferredModelFile.Path, model.PreferredModelFile.Size)
-		} else {
-			modelsInfo += "  No preferred model file found\n"
-		}
-
-		if model.ReadmeContent != "" {
-			modelsInfo += fmt.Sprintf("  README: %s\n", model.ReadmeContent)
-		}
-
-		if model.ProcessingError != "" {
-			modelsInfo += fmt.Sprintf("  Processing Error: %s\n", model.ProcessingError)
-		}
-
-		modelsInfo += "\n"
-	}
-
-	fragment = fragment.AddMessage("user", modelsInfo)
-
-	fragment = fragment.AddMessage("user", "Based on your analysis, select the top 5 most interesting models and provide a brief explanation for each selection. Also, create a filtered SearchResult with only the selected models. Return just a list of repositories IDs, you will later be asked to output it as a JSON array with the json tool.")
-
-	// Get a response
-	newFragment, err := llm.Ask(ctx, fragment)
-	if err != nil {
-		return nil, err
-	}
-
-	fmt.Println(newFragment.LastMessage().Content)
-	repositories := struct {
-		Repositories []string `json:"repositories"`
-	}{}
-
-	s := structures.Structure{
-		Schema: jsonschema.Definition{
-			Type:                 jsonschema.Object,
-			AdditionalProperties: false,
-			Properties: map[string]jsonschema.Definition{
-				"repositories": {
-					Type:        jsonschema.Array,
-					Items:       &jsonschema.Definition{Type: jsonschema.String},
-					Description: "The trending repositories IDs",
-				},
-			},
-			Required: []string{"repositories"},
-		},
-		Object: &repositories,
-	}
-
-	err = newFragment.ExtractStructure(ctx, llm, s)
-	if err != nil {
-		return nil, err
-	}
-
-	filteredModels := []ProcessedModel{}
-	for _, m := range searchResult.Models {
-		if slices.Contains(repositories.Repositories, m.ModelID) {
-			filteredModels = append(filteredModels, m)
-		}
-	}
-
-	return filteredModels, nil
-}
-
-// ModelMetadata represents extracted metadata from a model
-type ModelMetadata struct {
-	Tags    []string `json:"tags"`
-	License string   `json:"license"`
-}
-
-// extractModelMetadata extracts tags and license from model README and documentation
-func extractModelMetadata(ctx context.Context, model ProcessedModel) ([]string, string, error) {
-	// Create a conversation fragment
-	fragment := cogito.NewEmptyFragment().
-		AddMessage("user",
-			`Your task is to extract metadata from an AI model's README and documentation. You will be provided with:
-1. Model information (ID, author, description)
-2. README content
-
-You need to extract:
-1. **Tags**: An array of relevant tags that describe the model. Use common tags from the gallery such as:
-   - llm, gguf, gpu, cpu, multimodal, image-to-text, text-to-text, text-to-speech, tts
-   - thinking, reasoning, chat, instruction-tuned, code, vision
-   - Model family names (e.g., llama, qwen, mistral, gemma) if applicable
-   - Any other relevant descriptive tags
-   Select 3-8 most relevant tags.
-
-2. **License**: The license identifier (e.g., "apache-2.0", "mit", "llama2", "gpl-3.0", "bsd", "cc-by-4.0").
-   If no license is found, return an empty string.
-
-Return the extracted metadata in a structured format.`)
-
-	// Add model information
-	modelInfo := "Model Information:\n"
-	modelInfo += fmt.Sprintf("  ID: %s\n", model.ModelID)
-	modelInfo += fmt.Sprintf("  Author: %s\n", model.Author)
-	modelInfo += fmt.Sprintf("  Downloads: %d\n", model.Downloads)
-	if model.ReadmeContent != "" {
-		modelInfo += fmt.Sprintf("  README Content:\n%s\n", model.ReadmeContent)
-	} else if model.ReadmeContentPreview != "" {
-		modelInfo += fmt.Sprintf("  README Preview: %s\n", model.ReadmeContentPreview)
-	}
-
-	fragment = fragment.AddMessage("user", modelInfo)
-	fragment = fragment.AddMessage("user", "Extract the tags and license from the model information. Return the metadata as a JSON object with 'tags' (array of strings) and 'license' (string).")
-
-	// Get a response
-	newFragment, err := llm.Ask(ctx, fragment)
-	if err != nil {
-		return nil, "", err
-	}
-
-	// Extract structured metadata
-	metadata := ModelMetadata{}
-
-	s := structures.Structure{
-		Schema: jsonschema.Definition{
-			Type:                 jsonschema.Object,
-			AdditionalProperties: false,
-			Properties: map[string]jsonschema.Definition{
-				"tags": {
-					Type:        jsonschema.Array,
-					Items:       &jsonschema.Definition{Type: jsonschema.String},
-					Description: "Array of relevant tags describing the model",
-				},
-				"license": {
-					Type:        jsonschema.String,
-					Description: "License identifier (e.g., apache-2.0, mit, llama2). Empty string if not found.",
-				},
-			},
-			Required: []string{"tags", "license"},
-		},
-		Object: &metadata,
-	}
-
-	err = newFragment.ExtractStructure(ctx, llm, s)
-	if err != nil {
-		return nil, "", err
-	}
-
-	return metadata.Tags, metadata.License, nil
-}
-
-// extractIconFromReadme scans the README content for image URLs and returns the first suitable icon URL found
-func extractIconFromReadme(readmeContent string) string {
-	if readmeContent == "" {
-		return ""
-	}
-
-	// Regular expressions to match image URLs in various formats (case-insensitive)
-	// Match markdown image syntax: ![alt](url) - case insensitive extensions
-	markdownImageRegex := regexp.MustCompile(`(?i)!\[[^\]]*\]\(([^)]+\.(png|jpg|jpeg|svg|webp|gif))\)`)
-	// Match HTML img tags: <img src="url">
-	htmlImageRegex := regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+\.(png|jpg|jpeg|svg|webp|gif))["']`)
-	// Match plain URLs ending with image extensions
-	plainImageRegex := regexp.MustCompile(`(?i)https?://[^\s<>"']+\.(png|jpg|jpeg|svg|webp|gif)`)
-
-	// Try markdown format first
-	matches := markdownImageRegex.FindStringSubmatch(readmeContent)
-	if len(matches) > 1 && matches[1] != "" {
-		url := strings.TrimSpace(matches[1])
-		// Prefer HuggingFace CDN URLs or absolute URLs
-		if strings.HasPrefix(strings.ToLower(url), "http") {
-			return url
-		}
-	}
-
-	// Try HTML img tags
-	matches = htmlImageRegex.FindStringSubmatch(readmeContent)
-	if len(matches) > 1 && matches[1] != "" {
-		url := strings.TrimSpace(matches[1])
-		if strings.HasPrefix(strings.ToLower(url), "http") {
-			return url
-		}
-	}
-
-	// Try plain URLs
-	matches = plainImageRegex.FindStringSubmatch(readmeContent)
-	if len(matches) > 0 {
-		url := strings.TrimSpace(matches[0])
-		if strings.HasPrefix(strings.ToLower(url), "http") {
-			return url
-		}
-	}
-
-	return ""
-}
-
-// getHuggingFaceAvatarURL attempts to get the HuggingFace avatar URL for a user
-func getHuggingFaceAvatarURL(author string) string {
-	if author == "" {
-		return ""
-	}
-
-	// Try to fetch user info from HuggingFace API
-	// HuggingFace API endpoint: https://huggingface.co/api/users/{username}
-	baseURL := "https://huggingface.co"
-	userURL := fmt.Sprintf("%s/api/users/%s", baseURL, author)
-
-	req, err := http.NewRequest("GET", userURL, nil)
-	if err != nil {
-		return ""
-	}
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return ""
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return ""
-	}
-
-	// Parse the response to get avatar URL
-	var userInfo map[string]interface{}
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return ""
-	}
-
-	if err := json.Unmarshal(body, &userInfo); err != nil {
-		return ""
-	}
-
-	// Try to extract avatar URL from response
-	if avatar, ok := userInfo["avatarUrl"].(string); ok && avatar != "" {
-		return avatar
-	}
-	if avatar, ok := userInfo["avatar"].(string); ok && avatar != "" {
-		return avatar
-	}
-
-	return ""
-}
-
-// extractModelIcon extracts icon URL from README or falls back to HuggingFace avatar
-func extractModelIcon(model ProcessedModel) string {
-	// First, try to extract icon from README
-	if icon := extractIconFromReadme(model.ReadmeContent); icon != "" {
-		return icon
-	}
-
-	// Fallback: Try to get HuggingFace user avatar
-	if model.Author != "" {
-		if avatar := getHuggingFaceAvatarURL(model.Author); avatar != "" {
-			return avatar
-		}
-	}
-
-	return ""
-}

.github/gallery-agent/gallery.go

@@ -1,200 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/ghodss/yaml"
-	"github.com/mudler/LocalAI/core/gallery/importers"
-)
-
-func formatTextContent(text string) string {
-	return formatTextContentWithIndent(text, 4, 6)
-}
-
-// formatTextContentWithIndent formats text content with specified base and list item indentation
-func formatTextContentWithIndent(text string, baseIndent int, listItemIndent int) string {
-	var formattedLines []string
-	lines := strings.Split(text, "\n")
-	for _, line := range lines {
-		trimmed := strings.TrimRight(line, " \t\r")
-		if trimmed == "" {
-			// Keep empty lines as empty (no indentation)
-			formattedLines = append(formattedLines, "")
-		} else {
-			// Preserve relative indentation from yaml.Marshal output
-			// Count existing leading spaces to preserve relative structure
-			leadingSpaces := len(trimmed) - len(strings.TrimLeft(trimmed, " \t"))
-			trimmedStripped := strings.TrimLeft(trimmed, " \t")
-
-			var totalIndent int
-			if strings.HasPrefix(trimmedStripped, "-") {
-				// List items: use listItemIndent (ignore existing leading spaces)
-				totalIndent = listItemIndent
-			} else {
-				// Regular lines: use baseIndent + preserve relative indentation
-				// This handles both top-level keys (leadingSpaces=0) and nested properties (leadingSpaces>0)
-				totalIndent = baseIndent + leadingSpaces
-			}
-
-			indentStr := strings.Repeat(" ", totalIndent)
-			formattedLines = append(formattedLines, indentStr+trimmedStripped)
-		}
-	}
-	formattedText := strings.Join(formattedLines, "\n")
-	// Remove any trailing spaces from the formatted description
-	formattedText = strings.TrimRight(formattedText, " \t")
-	return formattedText
-}
-
-// generateYAMLEntry generates a YAML entry for a model using the specified anchor
-func generateYAMLEntry(model ProcessedModel, quantization string) string {
-	modelConfig, err := importers.DiscoverModelConfig("https://huggingface.co/"+model.ModelID, json.RawMessage(`{ "quantization": "`+quantization+`"}`))
-	if err != nil {
-		panic(err)
-	}
-
-	// Extract model name from ModelID
-	parts := strings.Split(model.ModelID, "/")
-	modelName := model.ModelID
-	if len(parts) > 0 {
-		modelName = strings.ToLower(parts[len(parts)-1])
-	}
-	// Remove common suffixes
-	modelName = strings.ReplaceAll(modelName, "-gguf", "")
-	modelName = strings.ReplaceAll(modelName, "-q4_k_m", "")
-	modelName = strings.ReplaceAll(modelName, "-q4_k_s", "")
-	modelName = strings.ReplaceAll(modelName, "-q3_k_m", "")
-	modelName = strings.ReplaceAll(modelName, "-q2_k", "")
-
-	description := model.ReadmeContent
-	if description == "" {
-		description = fmt.Sprintf("AI model: %s", modelName)
-	}
-
-	// Clean up description to prevent YAML linting issues
-	description = cleanTextContent(description)
-	formattedDescription := formatTextContent(description)
-
-	configFile := formatTextContent(modelConfig.ConfigFile)
-
-	filesYAML, _ := yaml.Marshal(modelConfig.Files)
-
-	// Files section: list items need 4 spaces (not 6), since files: is at 2 spaces
-	files := formatTextContentWithIndent(string(filesYAML), 4, 4)
-
-	// Build metadata sections
-	var metadataSections []string
-
-	// Add license if present
-	if model.License != "" {
-		metadataSections = append(metadataSections, fmt.Sprintf(`  license: "%s"`, model.License))
-	}
-
-	// Add tags if present
-	if len(model.Tags) > 0 {
-		tagsYAML, _ := yaml.Marshal(model.Tags)
-		tagsFormatted := formatTextContentWithIndent(string(tagsYAML), 4, 4)
-		tagsFormatted = strings.TrimRight(tagsFormatted, "\n")
-		metadataSections = append(metadataSections, fmt.Sprintf("  tags:\n%s", tagsFormatted))
-	}
-
-	// Add icon if present
-	if model.Icon != "" {
-		metadataSections = append(metadataSections, fmt.Sprintf(`  icon: %s`, model.Icon))
-	}
-
-	// Build the metadata block
-	metadataBlock := ""
-	if len(metadataSections) > 0 {
-		metadataBlock = strings.Join(metadataSections, "\n") + "\n"
-	}
-
-	yamlTemplate := ""
-	yamlTemplate = `- name: "%s"
-  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
-  urls:
-    - https://huggingface.co/%s
-  description: |
-%s%s
-  overrides:
-%s
-  files:
-%s`
-	// Trim trailing newlines from formatted sections to prevent extra blank lines
-	formattedDescription = strings.TrimRight(formattedDescription, "\n")
-	configFile = strings.TrimRight(configFile, "\n")
-	files = strings.TrimRight(files, "\n")
-	// Add newline before metadata block if present
-	if metadataBlock != "" {
-		metadataBlock = "\n" + strings.TrimRight(metadataBlock, "\n")
-	}
-	return fmt.Sprintf(yamlTemplate,
-		modelName,
-		model.ModelID,
-		formattedDescription,
-		metadataBlock,
-		configFile,
-		files,
-	)
-}
-
-// generateYAMLForModels generates YAML entries for selected models and appends to index.yaml
-func generateYAMLForModels(ctx context.Context, models []ProcessedModel, quantization string) error {
-
-	// Generate YAML entries for each model
-	var yamlEntries []string
-	for _, model := range models {
-		fmt.Printf("Generating YAML entry for model: %s\n", model.ModelID)
-
-		// Generate YAML entry
-		yamlEntry := generateYAMLEntry(model, quantization)
-		yamlEntries = append(yamlEntries, yamlEntry)
-	}
-
-	// Prepend to index.yaml (write at the top)
-	if len(yamlEntries) > 0 {
-		indexPath := getGalleryIndexPath()
-		fmt.Printf("Prepending YAML entries to %s...\n", indexPath)
-
-		// Read current content
-		content, err := os.ReadFile(indexPath)
-		if err != nil {
-			return fmt.Errorf("failed to read %s: %w", indexPath, err)
-		}
-
-		existingContent := string(content)
-		yamlBlock := strings.Join(yamlEntries, "\n")
-
-		// Check if file starts with "---"
-		var newContent string
-		if strings.HasPrefix(existingContent, "---\n") {
-			// File starts with "---", prepend new entries after it
-			restOfContent := strings.TrimPrefix(existingContent, "---\n")
-			// Ensure proper spacing: "---\n" + new entries + "\n" + rest of content
-			newContent = "---\n" + yamlBlock + "\n" + restOfContent
-		} else if strings.HasPrefix(existingContent, "---") {
-			// File starts with "---" but no newline after
-			restOfContent := strings.TrimPrefix(existingContent, "---")
-			newContent = "---\n" + yamlBlock + "\n" + strings.TrimPrefix(restOfContent, "\n")
-		} else {
-			// No "---" at start, prepend new entries at the very beginning
-			// Trim leading whitespace from existing content
-			existingContent = strings.TrimLeft(existingContent, " \t\n\r")
-			newContent = yamlBlock + "\n" + existingContent
-		}
-
-		// Write back to file
-		err = os.WriteFile(indexPath, []byte(newContent), 0644)
-		if err != nil {
-			return fmt.Errorf("failed to write %s: %w", indexPath, err)
-		}
-
-		fmt.Printf("Successfully prepended %d models to %s\n", len(yamlEntries), indexPath)
-	}
-
-	return nil
-}

.github/gallery-agent/main.go

@@ -1,383 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
-	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
-)
-
-// ProcessedModelFile represents a processed model file with additional metadata
-type ProcessedModelFile struct {
-	Path     string `json:"path"`
-	Size     int64  `json:"size"`
-	SHA256   string `json:"sha256"`
-	IsReadme bool   `json:"is_readme"`
-	FileType string `json:"file_type"` // "model", "readme", "other"
-}
-
-// ProcessedModel represents a processed model with all gathered metadata
-type ProcessedModel struct {
-	ModelID                 string               `json:"model_id"`
-	Author                  string               `json:"author"`
-	Downloads               int                  `json:"downloads"`
-	LastModified            string               `json:"last_modified"`
-	Files                   []ProcessedModelFile `json:"files"`
-	PreferredModelFile      *ProcessedModelFile  `json:"preferred_model_file,omitempty"`
-	ReadmeFile              *ProcessedModelFile  `json:"readme_file,omitempty"`
-	ReadmeContent           string               `json:"readme_content,omitempty"`
-	ReadmeContentPreview    string               `json:"readme_content_preview,omitempty"`
-	QuantizationPreferences []string             `json:"quantization_preferences"`
-	ProcessingError         string               `json:"processing_error,omitempty"`
-	Tags                    []string             `json:"tags,omitempty"`
-	License                 string               `json:"license,omitempty"`
-	Icon                    string               `json:"icon,omitempty"`
-}
-
-// SearchResult represents the complete result of searching and processing models
-type SearchResult struct {
-	SearchTerm       string           `json:"search_term"`
-	Limit            int              `json:"limit"`
-	Quantization     string           `json:"quantization"`
-	TotalModelsFound int              `json:"total_models_found"`
-	Models           []ProcessedModel `json:"models"`
-	FormattedOutput  string           `json:"formatted_output"`
-}
-
-// AddedModelSummary represents a summary of models added to the gallery
-type AddedModelSummary struct {
-	SearchTerm     string   `json:"search_term"`
-	TotalFound     int      `json:"total_found"`
-	ModelsAdded    int      `json:"models_added"`
-	AddedModelIDs  []string `json:"added_model_ids"`
-	AddedModelURLs []string `json:"added_model_urls"`
-	Quantization   string   `json:"quantization"`
-	ProcessingTime string   `json:"processing_time"`
-}
-
-func main() {
-	startTime := time.Now()
-
-	// Check for synthetic mode
-	syntheticMode := os.Getenv("SYNTHETIC_MODE")
-	if syntheticMode == "true" || syntheticMode == "1" {
-		fmt.Println("Running in SYNTHETIC MODE - generating random test data")
-		err := runSyntheticMode()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error in synthetic mode: %v\n", err)
-			os.Exit(1)
-		}
-		return
-	}
-
-	// Get configuration from environment variables
-	searchTerm := os.Getenv("SEARCH_TERM")
-	if searchTerm == "" {
-		searchTerm = "GGUF"
-	}
-
-	limitStr := os.Getenv("LIMIT")
-	if limitStr == "" {
-		limitStr = "5"
-	}
-	limit, err := strconv.Atoi(limitStr)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error parsing LIMIT: %v\n", err)
-		os.Exit(1)
-	}
-
-	quantization := os.Getenv("QUANTIZATION")
-
-	maxModels := os.Getenv("MAX_MODELS")
-	if maxModels == "" {
-		maxModels = "1"
-	}
-	maxModelsInt, err := strconv.Atoi(maxModels)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error parsing MAX_MODELS: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Print configuration
-	fmt.Printf("Gallery Agent Configuration:\n")
-	fmt.Printf("  Search Term: %s\n", searchTerm)
-	fmt.Printf("  Limit: %d\n", limit)
-	fmt.Printf("  Quantization: %s\n", quantization)
-	fmt.Printf("  Max Models to Add: %d\n", maxModelsInt)
-	fmt.Printf("  Gallery Index Path: %s\n", os.Getenv("GALLERY_INDEX_PATH"))
-	fmt.Println()
-
-	result, err := searchAndProcessModels(searchTerm, limit, quantization)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
-		os.Exit(1)
-	}
-
-	fmt.Println(result.FormattedOutput)
-	var models []ProcessedModel
-
-	if len(result.Models) > 1 {
-		fmt.Println("More than one model found (", len(result.Models), "), using AI agent to select the most interesting models")
-		for _, model := range result.Models {
-			fmt.Println("Model: ", model.ModelID)
-		}
-		// Use AI agent to select the most interesting models
-		fmt.Println("Using AI agent to select the most interesting models...")
-		models, err = selectMostInterestingModels(context.Background(), result)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error in model selection: %v\n", err)
-			// Continue with original result if selection fails
-			models = result.Models
-		}
-	} else if len(result.Models) == 1 {
-		models = result.Models
-		fmt.Println("Only one model found, using it directly")
-	}
-
-	fmt.Print(models)
-
-	// Filter out models that already exist in the gallery
-	fmt.Println("Filtering out existing models...")
-	models, err = filterExistingModels(models)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error filtering existing models: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Limit to maxModelsInt after filtering
-	if len(models) > maxModelsInt {
-		models = models[:maxModelsInt]
-	}
-
-	// Track added models for summary
-	var addedModelIDs []string
-	var addedModelURLs []string
-
-	// Generate YAML entries and append to gallery/index.yaml
-	if len(models) > 0 {
-		for _, model := range models {
-			addedModelIDs = append(addedModelIDs, model.ModelID)
-			// Generate Hugging Face URL for the model
-			modelURL := fmt.Sprintf("https://huggingface.co/%s", model.ModelID)
-			addedModelURLs = append(addedModelURLs, modelURL)
-		}
-		fmt.Println("Generating YAML entries for selected models...")
-		err = generateYAMLForModels(context.Background(), models, quantization)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error generating YAML entries: %v\n", err)
-			os.Exit(1)
-		}
-	} else {
-		fmt.Println("No new models to add to the gallery.")
-	}
-
-	// Create and write summary
-	processingTime := time.Since(startTime).String()
-	summary := AddedModelSummary{
-		SearchTerm:     searchTerm,
-		TotalFound:     result.TotalModelsFound,
-		ModelsAdded:    len(addedModelIDs),
-		AddedModelIDs:  addedModelIDs,
-		AddedModelURLs: addedModelURLs,
-		Quantization:   quantization,
-		ProcessingTime: processingTime,
-	}
-
-	// Write summary to file
-	summaryData, err := json.MarshalIndent(summary, "", "  ")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error marshaling summary: %v\n", err)
-	} else {
-		err = os.WriteFile("gallery-agent-summary.json", summaryData, 0644)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error writing summary file: %v\n", err)
-		} else {
-			fmt.Printf("Summary written to gallery-agent-summary.json\n")
-		}
-	}
-}
-
-func searchAndProcessModels(searchTerm string, limit int, quantization string) (*SearchResult, error) {
-	client := hfapi.NewClient()
-	var outputBuilder strings.Builder
-
-	fmt.Println("Searching for models...")
-	// Initialize the result struct
-	result := &SearchResult{
-		SearchTerm:   searchTerm,
-		Limit:        limit,
-		Quantization: quantization,
-		Models:       []ProcessedModel{},
-	}
-
-	models, err := client.GetLatest(searchTerm, limit)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch models: %w", err)
-	}
-
-	fmt.Println("Models found:", len(models))
-	result.TotalModelsFound = len(models)
-
-	if len(models) == 0 {
-		outputBuilder.WriteString("No models found.\n")
-		result.FormattedOutput = outputBuilder.String()
-		return result, nil
-	}
-
-	outputBuilder.WriteString(fmt.Sprintf("Found %d models matching '%s':\n\n", len(models), searchTerm))
-
-	// Process each model
-	for i, model := range models {
-		outputBuilder.WriteString(fmt.Sprintf("%d. Processing Model: %s\n", i+1, model.ModelID))
-		outputBuilder.WriteString(fmt.Sprintf("   Author: %s\n", model.Author))
-		outputBuilder.WriteString(fmt.Sprintf("   Downloads: %d\n", model.Downloads))
-		outputBuilder.WriteString(fmt.Sprintf("   Last Modified: %s\n", model.LastModified))
-
-		// Initialize processed model struct
-		processedModel := ProcessedModel{
-			ModelID:                 model.ModelID,
-			Author:                  model.Author,
-			Downloads:               model.Downloads,
-			LastModified:            model.LastModified,
-			QuantizationPreferences: []string{quantization, "Q4_K_M", "Q4_K_S", "Q3_K_M", "Q2_K"},
-		}
-
-		// Get detailed model information
-		details, err := client.GetModelDetails(model.ModelID)
-		if err != nil {
-			errorMsg := fmt.Sprintf("   Error getting model details: %v\n", err)
-			outputBuilder.WriteString(errorMsg)
-			processedModel.ProcessingError = err.Error()
-			result.Models = append(result.Models, processedModel)
-			continue
-		}
-
-		// Define quantization preferences (in order of preference)
-		quantizationPreferences := []string{quantization, "Q4_K_M", "Q4_K_S", "Q3_K_M", "Q2_K"}
-
-		// Find preferred model file
-		preferredModelFile := hfapi.FindPreferredModelFile(details.Files, quantizationPreferences)
-
-		// Process files
-		processedFiles := make([]ProcessedModelFile, len(details.Files))
-		for j, file := range details.Files {
-			fileType := "other"
-			if file.IsReadme {
-				fileType = "readme"
-			} else if preferredModelFile != nil && file.Path == preferredModelFile.Path {
-				fileType = "model"
-			}
-
-			processedFiles[j] = ProcessedModelFile{
-				Path:     file.Path,
-				Size:     file.Size,
-				SHA256:   file.SHA256,
-				IsReadme: file.IsReadme,
-				FileType: fileType,
-			}
-		}
-
-		processedModel.Files = processedFiles
-
-		// Set preferred model file
-		if preferredModelFile != nil {
-			for _, file := range processedFiles {
-				if file.Path == preferredModelFile.Path {
-					processedModel.PreferredModelFile = &file
-					break
-				}
-			}
-		}
-
-		// Print file information
-		outputBuilder.WriteString(fmt.Sprintf("   Files found: %d\n", len(details.Files)))
-
-		if preferredModelFile != nil {
-			outputBuilder.WriteString(fmt.Sprintf("   Preferred Model File: %s (SHA256: %s)\n",
-				preferredModelFile.Path,
-				preferredModelFile.SHA256))
-		} else {
-			outputBuilder.WriteString(fmt.Sprintf("   No model file found with quantization preferences: %v\n", quantizationPreferences))
-		}
-
-		if details.ReadmeFile != nil {
-			outputBuilder.WriteString(fmt.Sprintf("   README File: %s\n", details.ReadmeFile.Path))
-
-			// Find and set readme file
-			for _, file := range processedFiles {
-				if file.IsReadme {
-					processedModel.ReadmeFile = &file
-					break
-				}
-			}
-
-			fmt.Println("Getting real readme for", model.ModelID, "waiting...")
-			// Use agent to get the real readme and prepare the model description
-			readmeContent, err := getRealReadme(context.Background(), model.ModelID)
-			if err == nil {
-				processedModel.ReadmeContent = readmeContent
-				processedModel.ReadmeContentPreview = truncateString(readmeContent, 200)
-				outputBuilder.WriteString(fmt.Sprintf("   README Content Preview: %s\n",
-					processedModel.ReadmeContentPreview))
-			} else {
-				fmt.Printf("   Warning: Failed to get real readme: %v\n", err)
-			}
-			fmt.Println("Real readme got", readmeContent)
-
-			// Extract metadata (tags, license) from README using LLM
-			fmt.Println("Extracting metadata for", model.ModelID, "waiting...")
-			tags, license, err := extractModelMetadata(context.Background(), processedModel)
-			if err == nil {
-				processedModel.Tags = tags
-				processedModel.License = license
-				outputBuilder.WriteString(fmt.Sprintf("   Tags: %v\n", tags))
-				outputBuilder.WriteString(fmt.Sprintf("   License: %s\n", license))
-			} else {
-				fmt.Printf("   Warning: Failed to extract metadata: %v\n", err)
-			}
-
-			// Extract icon from README or use HuggingFace avatar
-			icon := extractModelIcon(processedModel)
-			if icon != "" {
-				processedModel.Icon = icon
-				outputBuilder.WriteString(fmt.Sprintf("   Icon: %s\n", icon))
-			}
-			// Get README content
-			// readmeContent, err := client.GetReadmeContent(model.ModelID, details.ReadmeFile.Path)
-			// if err == nil {
-			// 	processedModel.ReadmeContent = readmeContent
-			// 	processedModel.ReadmeContentPreview = truncateString(readmeContent, 200)
-			// 	outputBuilder.WriteString(fmt.Sprintf("   README Content Preview: %s\n",
-			// 		processedModel.ReadmeContentPreview))
-			// }
-		}
-
-		// Print all files with their checksums
-		outputBuilder.WriteString("   All Files:\n")
-		for _, file := range processedFiles {
-			outputBuilder.WriteString(fmt.Sprintf("     - %s (%s, %d bytes", file.Path, file.FileType, file.Size))
-			if file.SHA256 != "" {
-				outputBuilder.WriteString(fmt.Sprintf(", SHA256: %s", file.SHA256))
-			}
-			outputBuilder.WriteString(")\n")
-		}
-
-		outputBuilder.WriteString("\n")
-		result.Models = append(result.Models, processedModel)
-	}
-
-	result.FormattedOutput = outputBuilder.String()
-	return result, nil
-}
-
-func truncateString(s string, maxLen int) string {
-	if len(s) <= maxLen {
-		return s
-	}
-	return s[:maxLen] + "..."
-}

.github/gallery-agent/testing.go

@@ -1,224 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"math/rand"
-	"strings"
-	"time"
-)
-
-// runSyntheticMode generates synthetic test data and appends it to the gallery
-func runSyntheticMode() error {
-	generator := NewSyntheticDataGenerator()
-
-	// Generate a random number of synthetic models (1-3)
-	numModels := generator.rand.Intn(3) + 1
-	fmt.Printf("Generating %d synthetic models for testing...\n", numModels)
-
-	var models []ProcessedModel
-	for i := 0; i < numModels; i++ {
-		model := generator.GenerateProcessedModel()
-		models = append(models, model)
-		fmt.Printf("Generated synthetic model: %s\n", model.ModelID)
-	}
-
-	// Generate YAML entries and append to gallery/index.yaml
-	fmt.Println("Generating YAML entries for synthetic models...")
-	err := generateYAMLForModels(context.Background(), models, "Q4_K_M")
-	if err != nil {
-		return fmt.Errorf("error generating YAML entries: %w", err)
-	}
-
-	fmt.Printf("Successfully added %d synthetic models to the gallery for testing!\n", len(models))
-	return nil
-}
-
-// SyntheticDataGenerator provides methods to generate synthetic test data
-type SyntheticDataGenerator struct {
-	rand *rand.Rand
-}
-
-// NewSyntheticDataGenerator creates a new synthetic data generator
-func NewSyntheticDataGenerator() *SyntheticDataGenerator {
-	return &SyntheticDataGenerator{
-		rand: rand.New(rand.NewSource(time.Now().UnixNano())),
-	}
-}
-
-// GenerateProcessedModelFile creates a synthetic ProcessedModelFile
-func (g *SyntheticDataGenerator) GenerateProcessedModelFile() ProcessedModelFile {
-	fileTypes := []string{"model", "readme", "other"}
-	fileType := fileTypes[g.rand.Intn(len(fileTypes))]
-
-	var path string
-	var isReadme bool
-
-	switch fileType {
-	case "model":
-		path = fmt.Sprintf("model-%s.gguf", g.randomString(8))
-		isReadme = false
-	case "readme":
-		path = "README.md"
-		isReadme = true
-	default:
-		path = fmt.Sprintf("file-%s.txt", g.randomString(6))
-		isReadme = false
-	}
-
-	return ProcessedModelFile{
-		Path:     path,
-		Size:     int64(g.rand.Intn(1000000000) + 1000000), // 1MB to 1GB
-		SHA256:   g.randomSHA256(),
-		IsReadme: isReadme,
-		FileType: fileType,
-	}
-}
-
-// GenerateProcessedModel creates a synthetic ProcessedModel
-func (g *SyntheticDataGenerator) GenerateProcessedModel() ProcessedModel {
-	authors := []string{"microsoft", "meta", "google", "openai", "anthropic", "mistralai", "huggingface"}
-	modelNames := []string{"llama", "gpt", "claude", "mistral", "gemma", "phi", "qwen", "codellama"}
-
-	author := authors[g.rand.Intn(len(authors))]
-	modelName := modelNames[g.rand.Intn(len(modelNames))]
-	modelID := fmt.Sprintf("%s/%s-%s", author, modelName, g.randomString(6))
-
-	// Generate files
-	numFiles := g.rand.Intn(5) + 2 // 2-6 files
-	files := make([]ProcessedModelFile, numFiles)
-
-	// Ensure at least one model file and one readme
-	hasModelFile := false
-	hasReadme := false
-
-	for i := 0; i < numFiles; i++ {
-		files[i] = g.GenerateProcessedModelFile()
-		if files[i].FileType == "model" {
-			hasModelFile = true
-		}
-		if files[i].FileType == "readme" {
-			hasReadme = true
-		}
-	}
-
-	// Add required files if missing
-	if !hasModelFile {
-		modelFile := g.GenerateProcessedModelFile()
-		modelFile.FileType = "model"
-		modelFile.Path = fmt.Sprintf("%s-Q4_K_M.gguf", modelName)
-		files = append(files, modelFile)
-	}
-
-	if !hasReadme {
-		readmeFile := g.GenerateProcessedModelFile()
-		readmeFile.FileType = "readme"
-		readmeFile.Path = "README.md"
-		readmeFile.IsReadme = true
-		files = append(files, readmeFile)
-	}
-
-	// Find preferred model file
-	var preferredModelFile *ProcessedModelFile
-	for i := range files {
-		if files[i].FileType == "model" {
-			preferredModelFile = &files[i]
-			break
-		}
-	}
-
-	// Find readme file
-	var readmeFile *ProcessedModelFile
-	for i := range files {
-		if files[i].FileType == "readme" {
-			readmeFile = &files[i]
-			break
-		}
-	}
-
-	readmeContent := g.generateReadmeContent(modelName, author)
-
-	// Generate sample metadata
-	licenses := []string{"apache-2.0", "mit", "llama2", "gpl-3.0", "bsd", ""}
-	license := licenses[g.rand.Intn(len(licenses))]
-
-	sampleTags := []string{"llm", "gguf", "gpu", "cpu", "text-to-text", "chat", "instruction-tuned"}
-	numTags := g.rand.Intn(4) + 3 // 3-6 tags
-	tags := make([]string, numTags)
-	for i := 0; i < numTags; i++ {
-		tags[i] = sampleTags[g.rand.Intn(len(sampleTags))]
-	}
-	// Remove duplicates
-	tags = g.removeDuplicates(tags)
-
-	// Optionally include icon (50% chance)
-	icon := ""
-	if g.rand.Intn(2) == 0 {
-		icon = fmt.Sprintf("https://cdn-avatars.huggingface.co/v1/production/uploads/%s.png", g.randomString(24))
-	}
-
-	return ProcessedModel{
-		ModelID:                 modelID,
-		Author:                  author,
-		Downloads:               g.rand.Intn(1000000) + 1000,
-		LastModified:            g.randomDate(),
-		Files:                   files,
-		PreferredModelFile:      preferredModelFile,
-		ReadmeFile:              readmeFile,
-		ReadmeContent:           readmeContent,
-		ReadmeContentPreview:    truncateString(readmeContent, 200),
-		QuantizationPreferences: []string{"Q4_K_M", "Q4_K_S", "Q3_K_M", "Q2_K"},
-		ProcessingError:         "",
-		Tags:                    tags,
-		License:                 license,
-		Icon:                    icon,
-	}
-}
-
-// Helper methods for synthetic data generation
-func (g *SyntheticDataGenerator) randomString(length int) string {
-	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
-	b := make([]byte, length)
-	for i := range b {
-		b[i] = charset[g.rand.Intn(len(charset))]
-	}
-	return string(b)
-}
-
-func (g *SyntheticDataGenerator) randomSHA256() string {
-	const charset = "0123456789abcdef"
-	b := make([]byte, 64)
-	for i := range b {
-		b[i] = charset[g.rand.Intn(len(charset))]
-	}
-	return string(b)
-}
-
-func (g *SyntheticDataGenerator) randomDate() string {
-	now := time.Now()
-	daysAgo := g.rand.Intn(365) // Random date within last year
-	pastDate := now.AddDate(0, 0, -daysAgo)
-	return pastDate.Format("2006-01-02T15:04:05.000Z")
-}
-
-func (g *SyntheticDataGenerator) removeDuplicates(slice []string) []string {
-	keys := make(map[string]bool)
-	result := []string{}
-	for _, item := range slice {
-		if !keys[item] {
-			keys[item] = true
-			result = append(result, item)
-		}
-	}
-	return result
-}
-
-func (g *SyntheticDataGenerator) generateReadmeContent(modelName, author string) string {
-	templates := []string{
-		fmt.Sprintf("# %s Model\n\nThis is a %s model developed by %s. It's designed for various natural language processing tasks including text generation, question answering, and conversation.\n\n## Features\n\n- High-quality text generation\n- Efficient inference\n- Multiple quantization options\n- Easy to use with LocalAI\n\n## Usage\n\nUse this model with LocalAI for various AI tasks.", strings.Title(modelName), modelName, author),
-		fmt.Sprintf("# %s\n\nA powerful language model from %s. This model excels at understanding and generating human-like text across multiple domains.\n\n## Capabilities\n\n- Text completion\n- Code generation\n- Creative writing\n- Technical documentation\n\n## Model Details\n\n- Architecture: Transformer-based\n- Training: Large-scale supervised learning\n- Quantization: Available in multiple formats", strings.Title(modelName), author),
-		fmt.Sprintf("# %s Language Model\n\nDeveloped by %s, this model represents state-of-the-art performance in natural language understanding and generation.\n\n## Key Features\n\n- Multilingual support\n- Context-aware responses\n- Efficient memory usage\n- Fast inference speed\n\n## Applications\n\n- Chatbots and virtual assistants\n- Content generation\n- Code completion\n- Educational tools", strings.Title(modelName), author),
-	}
-
-	return templates[g.rand.Intn(len(templates))]
-}

.github/gallery-agent/tools.go

@@ -1,46 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
-	openai "github.com/sashabaranov/go-openai"
-	jsonschema "github.com/sashabaranov/go-openai/jsonschema"
-)
-
-// Get repository README from HF
-type HFReadmeTool struct {
-	client *hfapi.Client
-}
-
-func (s *HFReadmeTool) Execute(args map[string]any) (string, error) {
-	q, ok := args["repository"].(string)
-	if !ok {
-		return "", fmt.Errorf("no query")
-	}
-	readme, err := s.client.GetReadmeContent(q, "README.md")
-	if err != nil {
-		return "", err
-	}
-	return readme, nil
-}
-
-func (s *HFReadmeTool) Tool() openai.Tool {
-	return openai.Tool{
-		Type: openai.ToolTypeFunction,
-		Function: &openai.FunctionDefinition{
-			Name:        "hf_readme",
-			Description: "A tool to get the README content of a huggingface repository",
-			Parameters: jsonschema.Definition{
-				Type: jsonschema.Object,
-				Properties: map[string]jsonschema.Definition{
-					"repository": {
-						Type:        jsonschema.String,
-						Description: "The huggingface repository to get the README content of",
-					},
-				},
-				Required: []string{"repository"},
-			},
-		},
-	}
-}

.github/workflows/backend_build.yml

@@ -1,5 +1,5 @@
 ---
-name: 'build backend container images (reusable)'
+name: 'build python backend container images (reusable)'
 
 on:
   workflow_call:
@@ -53,11 +53,6 @@ on:
         description: 'Skip drivers'
         default: 'false'
         type: string
-      ubuntu-version:
-        description: 'Ubuntu version'
-        required: false
-        default: '2204'
-        type: string
     secrets:
       dockerUsername:
         required: false
@@ -102,7 +97,7 @@ jobs:
           && sudo apt-get install -y git
 
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Release space from worker
         if: inputs.runs-on == 'ubuntu-latest'
@@ -213,7 +208,6 @@ jobs:
             CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }}
             BASE_IMAGE=${{ inputs.base-image }}
             BACKEND=${{ inputs.backend }}
-            UBUNTU_VERSION=${{ inputs.ubuntu-version }}
           context: ${{ inputs.context }}
           file: ${{ inputs.dockerfile }}
           cache-from: type=gha
@@ -234,7 +228,6 @@ jobs:
             CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }}
             BASE_IMAGE=${{ inputs.base-image }}
             BACKEND=${{ inputs.backend }}
-            UBUNTU_VERSION=${{ inputs.ubuntu-version }}
           context: ${{ inputs.context }}
           file: ${{ inputs.dockerfile }}
           cache-from: type=gha

.github/workflows/backend_build_darwin.yml

@@ -50,7 +50,7 @@ jobs:
         go-version: ['${{ inputs.go-version }}']
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
 
@@ -74,7 +74,7 @@ jobs:
           BACKEND=${{ inputs.backend }} BUILD_TYPE=${{ inputs.build-type }} USE_PIP=${{ inputs.use-pip }} make build-darwin-${{ inputs.lang }}-backend
 
       - name: Upload ${{ inputs.backend }}.tar
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.backend }}-tar
           path: backend-images/${{ inputs.backend }}.tar
@@ -85,7 +85,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download ${{ inputs.backend }}.tar
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v5
         with:
           name: ${{ inputs.backend }}-tar
           path: .

.github/workflows/backend_pr.yml

@@ -17,7 +17,7 @@ jobs:
       has-backends-darwin: ${{ steps.set-matrix.outputs.has-backends-darwin }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Setup Bun
         uses: oven-sh/setup-bun@v2
@@ -52,7 +52,6 @@ jobs:
       dockerfile: ${{ matrix.dockerfile }}
       skip-drivers: ${{ matrix.skip-drivers }}
       context: ${{ matrix.context }}
-      ubuntu-version: ${{ matrix.ubuntu-version }}
     secrets:
       quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
@@ -70,7 +69,7 @@ jobs:
       tag-suffix: ${{ matrix.tag-suffix }}
       lang: ${{ matrix.lang || 'python' }}
       use-pip: ${{ matrix.backend == 'diffusers' }}
-      runs-on: "macos-latest"
+      runs-on: "macOS-14"
     secrets:
       quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}

.github/workflows/build-test.yaml

@@ -11,13 +11,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.25
+          go-version: 1.23
       - name: Run GoReleaser
         run: |
           make dev-dist
@@ -25,19 +25,19 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.25
+          go-version: 1.23
       - name: Build launcher for macOS ARM64
         run: |
           make build-launcher-darwin
           ls -liah dist
       - name: Upload macOS launcher artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: launcher-macos
           path: dist/
@@ -47,20 +47,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.25
+          go-version: 1.23
       - name: Build launcher for Linux
         run: |
           sudo apt-get update
           sudo apt-get install golang gcc libgl1-mesa-dev xorg-dev libxkbcommon-dev
           make build-launcher-linux
       - name: Upload Linux launcher artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: launcher-linux
           path: local-ai-launcher-linux.tar.xz

.github/workflows/bump_deps.yaml

@@ -1,10 +1,10 @@
-name: Bump Backend dependencies
+name: Bump dependencies
 on:
   schedule:
     - cron: 0 20 * * *
   workflow_dispatch:
 jobs:
-  bump-backends:
+  bump:
     strategy:
       fail-fast: false
       matrix:
@@ -31,7 +31,7 @@ jobs:
             file: "backend/go/piper/Makefile"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v5
       - name: Bump dependencies 🔧
         id: bump
         run: |
@@ -49,7 +49,7 @@ jobs:
           rm -rfv ${{ matrix.variable }}_message.txt
           rm -rfv ${{ matrix.variable }}_commit.txt
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v8
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.UPDATE_BOT_TOKEN }}
           push-to-fork: ci-forks/LocalAI

.github/workflows/bump_docs.yaml

@@ -1,10 +1,10 @@
-name: Bump Documentation
+name: Bump dependencies
 on:
   schedule:
     - cron: 0 20 * * *
   workflow_dispatch:
 jobs:
-  bump-docs:
+  bump:
     strategy:
       fail-fast: false
       matrix:
@@ -12,12 +12,12 @@ jobs:
           - repository: "mudler/LocalAI"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v5
       - name: Bump dependencies 🔧
         run: |
           bash .github/bump_docs.sh ${{ matrix.repository }}
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v8
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.UPDATE_BOT_TOKEN }}
           push-to-fork: ci-forks/LocalAI

.github/workflows/checksum_checker.yaml

@@ -15,7 +15,7 @@ jobs:
           && sudo add-apt-repository -y ppa:git-core/ppa \
           && sudo apt-get update \
           && sudo apt-get install -y git
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v5
       - name: Install dependencies
         run: |
           sudo apt-get update
@@ -35,7 +35,7 @@ jobs:
           sudo chmod 777 /hf_cache
           bash .github/checksum_checker.sh gallery/index.yaml
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v8
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.UPDATE_BOT_TOKEN }}
           push-to-fork: ci-forks/LocalAI

.github/workflows/dependabot_auto.yml

@@ -20,7 +20,7 @@ jobs:
           skip-commit-verification: true
 
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Approve a PR if not already approved
         run: |

.github/workflows/deploy-explorer.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - uses: actions/setup-go@v5
@@ -33,7 +33,7 @@ jobs:
         run: |
           CGO_ENABLED=0 make build
       - name: rm
-        uses: appleboy/ssh-action@v1.2.4
+        uses: appleboy/ssh-action@v1.2.2
         with:
             host: ${{ secrets.EXPLORER_SSH_HOST }}
             username: ${{ secrets.EXPLORER_SSH_USERNAME }}
@@ -53,7 +53,7 @@ jobs:
             rm: true
             target: ./local-ai
       - name: restarting
-        uses: appleboy/ssh-action@v1.2.4
+        uses: appleboy/ssh-action@v1.2.2
         with:
             host: ${{ secrets.EXPLORER_SSH_HOST }}
             username: ${{ secrets.EXPLORER_SSH_USERNAME }}

.github/workflows/gallery-agent.yaml

@@ -1,132 +0,0 @@
-name: Gallery Agent
-on:
-
-  schedule:
-    - cron: '0 */3 * * *'  # Run every 4 hours
-  workflow_dispatch:
-    inputs:
-      search_term:
-        description: 'Search term for models'
-        required: false
-        default: 'GGUF'
-        type: string
-      limit:
-        description: 'Maximum number of models to process'
-        required: false
-        default: '15'
-        type: string
-      quantization:
-        description: 'Preferred quantization format'
-        required: false
-        default: 'Q4_K_M'
-        type: string
-      max_models:
-        description: 'Maximum number of models to add to the gallery'
-        required: false
-        default: '1'
-        type: string
-jobs:
-  gallery-agent:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v6
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.21'
-      - name: Proto Dependencies
-        run: |
-          # Install protoc
-          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
-          unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
-          rm protoc.zip
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
-          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
-          PATH="$PATH:$HOME/go/bin" make protogen-go
-      - uses: mudler/localai-github-action@v1.1
-        with:
-          model: 'https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF'
-
-      - name: Run gallery agent
-        env:
-          #OPENAI_MODEL: ${{ secrets.OPENAI_MODEL }}
-          OPENAI_MODE: Qwen_Qwen3-1.7B-GGUF
-          OPENAI_BASE_URL: "http://localhost:8080"
-          OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
-          #OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
-          SEARCH_TERM: ${{ github.event.inputs.search_term || 'GGUF' }}
-          LIMIT: ${{ github.event.inputs.limit || '15' }}
-          QUANTIZATION: ${{ github.event.inputs.quantization || 'Q4_K_M' }}
-          MAX_MODELS: ${{ github.event.inputs.max_models || '1' }}
-        run: |
-          export GALLERY_INDEX_PATH=$PWD/gallery/index.yaml
-          go run ./.github/gallery-agent
-
-      - name: Check for changes
-        id: check_changes
-        run: |
-          if git diff --quiet gallery/index.yaml; then
-            echo "changes=false" >> $GITHUB_OUTPUT
-            echo "No changes detected in gallery/index.yaml"
-          else
-            echo "changes=true" >> $GITHUB_OUTPUT
-            echo "Changes detected in gallery/index.yaml"
-            git diff gallery/index.yaml
-          fi
-
-      - name: Read gallery agent summary
-        id: read_summary
-        if: steps.check_changes.outputs.changes == 'true'
-        run: |
-          if [ -f "./gallery-agent-summary.json" ]; then
-            echo "summary_exists=true" >> $GITHUB_OUTPUT
-            # Extract summary data using jq
-            echo "search_term=$(jq -r '.search_term' ./gallery-agent-summary.json)" >> $GITHUB_OUTPUT
-            echo "total_found=$(jq -r '.total_found' ./gallery-agent-summary.json)" >> $GITHUB_OUTPUT
-            echo "models_added=$(jq -r '.models_added' ./gallery-agent-summary.json)" >> $GITHUB_OUTPUT
-            echo "quantization=$(jq -r '.quantization' ./gallery-agent-summary.json)" >> $GITHUB_OUTPUT
-            echo "processing_time=$(jq -r '.processing_time' ./gallery-agent-summary.json)" >> $GITHUB_OUTPUT
-            
-            # Create a formatted list of added models with URLs
-            added_models=$(jq -r 'range(0; .added_model_ids | length) as $i | "- [\(.added_model_ids[$i])](\(.added_model_urls[$i]))"' ./gallery-agent-summary.json | tr '\n' '\n')
-            echo "added_models<<EOF" >> $GITHUB_OUTPUT
-            echo "$added_models" >> $GITHUB_OUTPUT
-            echo "EOF" >> $GITHUB_OUTPUT
-            rm -f ./gallery-agent-summary.json
-          else
-            echo "summary_exists=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Create Pull Request
-        if: steps.check_changes.outputs.changes == 'true'
-        uses: peter-evans/create-pull-request@v8
-        with:
-          token: ${{ secrets.UPDATE_BOT_TOKEN }}
-          push-to-fork: ci-forks/LocalAI
-          commit-message: 'chore(model gallery): :robot: add new models via gallery agent'
-          title: 'chore(model gallery): :robot: add ${{ steps.read_summary.outputs.models_added || 0 }} new models via gallery agent'
-          # Branch has to be unique so PRs are not overriding each other
-          branch-suffix: timestamp
-          body: |
-            This PR was automatically created by the gallery agent workflow.
-            
-            **Summary:**
-            - **Search Term:** ${{ steps.read_summary.outputs.search_term || github.event.inputs.search_term || 'GGUF' }}
-            - **Models Found:** ${{ steps.read_summary.outputs.total_found || 'N/A' }}
-            - **Models Added:** ${{ steps.read_summary.outputs.models_added || '0' }}
-            - **Quantization:** ${{ steps.read_summary.outputs.quantization || github.event.inputs.quantization || 'Q4_K_M' }}
-            - **Processing Time:** ${{ steps.read_summary.outputs.processing_time || 'N/A' }}
-            
-            **Added Models:**
-            ${{ steps.read_summary.outputs.added_models || '- No models added' }}
-            
-            **Workflow Details:**
-            - Triggered by: `${{ github.event_name }}`
-            - Run ID: `${{ github.run_id }}`
-            - Commit: `${{ github.sha }}`
-          signoff: true
-          delete-branch: true

.github/workflows/generate_grpc_cache.yaml

@@ -73,7 +73,7 @@ jobs:
         uses: docker/setup-buildx-action@master
 
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Cache GRPC
         uses: docker/build-push-action@v6

.github/workflows/generate_intel_image.yaml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         include:
           - base-image: intel/oneapi-basekit:2025.2.0-0-devel-ubuntu22.04
-            runs-on: 'arc-runner-set'
+            runs-on: 'ubuntu-latest'
             platforms: 'linux/amd64'
     runs-on: ${{matrix.runs-on}}
     steps:
@@ -43,7 +43,7 @@ jobs:
         uses: docker/setup-buildx-action@master
 
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Cache Intel images
         uses: docker/build-push-action@v6

.github/workflows/image-pr.yml

@@ -22,7 +22,6 @@ jobs:
       base-image: ${{ matrix.base-image }}
       grpc-base-image: ${{ matrix.grpc-base-image }}
       makeflags: ${{ matrix.makeflags }}
-      ubuntu-version: ${{ matrix.ubuntu-version }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -44,17 +43,6 @@ jobs:
             runs-on: 'ubuntu-latest'
             base-image: "ubuntu:22.04"
             makeflags: "--jobs=3 --output-sync=target"
-            ubuntu-version: '2204'
-          - build-type: 'cublas'
-            cuda-major-version: "13"
-            cuda-minor-version: "0"
-            platforms: 'linux/amd64'
-            tag-latest: 'false'
-            tag-suffix: '-gpu-nvidia-cuda-13'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:22.04"
-            makeflags: "--jobs=3 --output-sync=target"
-            ubuntu-version: '2204'
           - build-type: 'hipblas'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -63,7 +51,6 @@ jobs:
             grpc-base-image: "ubuntu:22.04"
             runs-on: 'ubuntu-latest'
             makeflags: "--jobs=3 --output-sync=target"
-            ubuntu-version: '2204'
           - build-type: 'sycl'
             platforms: 'linux/amd64'
             tag-latest: 'false'
@@ -72,23 +59,10 @@ jobs:
             tag-suffix: 'sycl'
             runs-on: 'ubuntu-latest'
             makeflags: "--jobs=3 --output-sync=target"
-            ubuntu-version: '2204'
           - build-type: 'vulkan'
-            platforms: 'linux/amd64'
+            platforms: 'linux/amd64,linux/arm64'
             tag-latest: 'false'
             tag-suffix: '-vulkan-core'
             runs-on: 'ubuntu-latest'
             base-image: "ubuntu:22.04"
             makeflags: "--jobs=4 --output-sync=target"
-            ubuntu-version: '2204'
-          - build-type: 'cublas'
-            cuda-major-version: "13"
-            cuda-minor-version: "0"
-            platforms: 'linux/arm64'
-            tag-latest: 'false'
-            tag-suffix: '-nvidia-l4t-arm64-cuda-13'
-            base-image: "ubuntu:24.04"
-            runs-on: 'ubuntu-24.04-arm'
-            makeflags: "--jobs=4 --output-sync=target"
-            skip-drivers: 'false'
-            ubuntu-version: '2404'

.github/workflows/image.yml

@@ -27,7 +27,6 @@ jobs:
       grpc-base-image: ${{ matrix.grpc-base-image }}
       aio: ${{ matrix.aio }}
       makeflags: ${{ matrix.makeflags }}
-      ubuntu-version: ${{ matrix.ubuntu-version }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -45,7 +44,6 @@ jobs:
             runs-on: 'ubuntu-latest'
             makeflags: "--jobs=3 --output-sync=target"
             aio: "-aio-gpu-hipblas"
-            ubuntu-version: '2204'
 
   core-image-build:
     uses: ./.github/workflows/image_build.yml
@@ -62,7 +60,6 @@ jobs:
       grpc-base-image: ${{ matrix.grpc-base-image }}
       makeflags: ${{ matrix.makeflags }}
       skip-drivers: ${{ matrix.skip-drivers }}
-      ubuntu-version: ${{ matrix.ubuntu-version }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -81,7 +78,6 @@ jobs:
             aio: "-aio-cpu"
             makeflags: "--jobs=4 --output-sync=target"
             skip-drivers: 'false'
-            ubuntu-version: '2204'
           - build-type: 'cublas'
             cuda-major-version: "11"
             cuda-minor-version: "7"
@@ -93,7 +89,6 @@ jobs:
             makeflags: "--jobs=4 --output-sync=target"
             skip-drivers: 'false'
             aio: "-aio-gpu-nvidia-cuda-11"
-            ubuntu-version: '2204'
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "0"
@@ -105,21 +100,8 @@ jobs:
             skip-drivers: 'false'
             makeflags: "--jobs=4 --output-sync=target"
             aio: "-aio-gpu-nvidia-cuda-12"
-            ubuntu-version: '2204'
-          - build-type: 'cublas'
-            cuda-major-version: "13"
-            cuda-minor-version: "0"
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-nvidia-cuda-13'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:22.04"
-            skip-drivers: 'false'
-            makeflags: "--jobs=4 --output-sync=target"
-            aio: "-aio-gpu-nvidia-cuda-13"
-            ubuntu-version: '2204'
           - build-type: 'vulkan'
-            platforms: 'linux/amd64'
+            platforms: 'linux/amd64,linux/arm64'
             tag-latest: 'auto'
             tag-suffix: '-gpu-vulkan'
             runs-on: 'ubuntu-latest'
@@ -127,7 +109,6 @@ jobs:
             skip-drivers: 'false'
             makeflags: "--jobs=4 --output-sync=target"
             aio: "-aio-gpu-vulkan"
-            ubuntu-version: '2204'
           - build-type: 'intel'
             platforms: 'linux/amd64'
             tag-latest: 'auto'
@@ -137,7 +118,6 @@ jobs:
             runs-on: 'ubuntu-latest'
             makeflags: "--jobs=3 --output-sync=target"
             aio: "-aio-gpu-intel"
-            ubuntu-version: '2204'
 
   gh-runner:
     uses: ./.github/workflows/image_build.yml
@@ -154,7 +134,6 @@ jobs:
       grpc-base-image: ${{ matrix.grpc-base-image }}
       makeflags: ${{ matrix.makeflags }}
       skip-drivers: ${{ matrix.skip-drivers }}
-      ubuntu-version: ${{ matrix.ubuntu-version }}
     secrets:
       dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -173,15 +152,3 @@ jobs:
             runs-on: 'ubuntu-24.04-arm'
             makeflags: "--jobs=4 --output-sync=target"
             skip-drivers: 'true'
-            ubuntu-version: "2204"
-          - build-type: 'cublas'
-            cuda-major-version: "13"
-            cuda-minor-version: "0"
-            platforms: 'linux/arm64'
-            tag-latest: 'auto'
-            tag-suffix: '-nvidia-l4t-arm64-cuda-13'
-            base-image: "ubuntu:24.04"
-            runs-on: 'ubuntu-24.04-arm'
-            makeflags: "--jobs=4 --output-sync=target"
-            skip-drivers: 'false'
-            ubuntu-version: '2404'

.github/workflows/image_build.yml

@@ -56,11 +56,6 @@ on:
         required: false
         default: ''
         type: string
-      ubuntu-version:
-        description: 'Ubuntu version'
-        required: false
-        default: '2204'
-        type: string
     secrets:
       dockerUsername:
         required: true
@@ -99,7 +94,7 @@ jobs:
           && sudo apt-get update \
           && sudo apt-get install -y git
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Release space from worker
         if: inputs.runs-on == 'ubuntu-latest'
@@ -243,7 +238,6 @@ jobs:
             GRPC_VERSION=v1.65.0
             MAKEFLAGS=${{ inputs.makeflags }}
             SKIP_DRIVERS=${{ inputs.skip-drivers }}
-            UBUNTU_VERSION=${{ inputs.ubuntu-version }}
           context: .
           file: ./Dockerfile
           cache-from: type=gha
@@ -271,7 +265,6 @@ jobs:
             GRPC_VERSION=v1.65.0
             MAKEFLAGS=${{ inputs.makeflags }}
             SKIP_DRIVERS=${{ inputs.skip-drivers }}
-            UBUNTU_VERSION=${{ inputs.ubuntu-version }}
           context: .
           file: ./Dockerfile
           cache-from: type=gha

.github/workflows/localaibot_automerge.yml

@@ -11,10 +11,10 @@ permissions:
 jobs:
   dependabot:
     runs-on: ubuntu-latest
-    if: ${{ github.actor == 'localai-bot' && !contains(github.event.pull_request.title, 'chore(model gallery):') }}
+    if: ${{ github.actor == 'localai-bot' }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Approve a PR if not already approved
         run: |

.github/workflows/notify-models.yaml

@@ -1,27 +1,22 @@
 name: Notifications for new models
 on:
-  pull_request_target:
+  pull_request:
      types:
        - closed
 
-permissions:
-  contents: read
-  pull-requests: read
-
 jobs:
   notify-discord:
     if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'area/ai-model')) }}
     env:
-        MODEL_NAME: gemma-3-12b-it-qat
+        MODEL_NAME: gemma-3-12b-it
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v5
       with:
         fetch-depth: 0 # needed to checkout all branches for this Action to work
-        ref: ${{ github.event.pull_request.head.sha }} # Checkout the PR head to get the actual changes
     - uses: mudler/localai-github-action@v1
       with:
-        model: 'gemma-3-12b-it-qat' # Any from models.localai.io, or from huggingface.com with: "huggingface://<repository>/file"
+        model: 'gemma-3-12b-it' # Any from models.localai.io, or from huggingface.com with: "huggingface://<repository>/file"
         # Check the PR diff using the current branch and the base branch of the PR
     - uses: GrantBirki/git-diff-action@v2.8.1
       id: git-diff-action
@@ -84,7 +79,7 @@ jobs:
         args: ${{ steps.summarize.outputs.message }}
     - name: Setup tmate session if fails
       if: ${{ failure() }}
-      uses: mxschmitt/action-tmate@v3.23
+      uses: mxschmitt/action-tmate@v3.22
       with:
         detached: true
         connect-timeout-seconds: 180
@@ -92,13 +87,12 @@ jobs:
   notify-twitter:
     if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'area/ai-model')) }}
     env:
-        MODEL_NAME: gemma-3-12b-it-qat
+        MODEL_NAME: gemma-3-12b-it
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v5
       with:
         fetch-depth: 0 # needed to checkout all branches for this Action to work
-        ref: ${{ github.event.pull_request.head.sha }} # Checkout the PR head to get the actual changes
     - name: Start LocalAI
       run: |
         echo "Starting LocalAI..."
@@ -167,7 +161,7 @@ jobs:
         TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }}
     - name: Setup tmate session if fails
       if: ${{ failure() }}
-      uses: mxschmitt/action-tmate@v3.23
+      uses: mxschmitt/action-tmate@v3.22
       with:
         detached: true
         connect-timeout-seconds: 180

.github/workflows/notify-releases.yaml

@@ -11,11 +11,10 @@ jobs:
         RELEASE_BODY: ${{ github.event.release.body }}
         RELEASE_TITLE: ${{ github.event.release.name }}
         RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
-        MODEL_NAME: gemma-3-12b-it-qat
     steps:
     - uses: mudler/localai-github-action@v1
       with:
-        model: 'gemma-3-12b-it-qat' # Any from models.localai.io, or from huggingface.com with: "huggingface://<repository>/file"
+        model: 'gemma-3-12b-it' # Any from models.localai.io, or from huggingface.com with: "huggingface://<repository>/file"
     - name: Summarize
       id: summarize
       run: |

.github/workflows/release.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Go
@@ -28,7 +28,7 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Go
@@ -46,7 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Go

.github/workflows/secscan.yaml

@@ -14,7 +14,7 @@ jobs:
       GO111MODULE: on
     steps:
       - name: Checkout Source
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         if: ${{ github.actor != 'dependabot[bot]' }}
       - name: Run Gosec Security Scanner
         if: ${{ github.actor != 'dependabot[bot]' }}
@@ -24,7 +24,7 @@ jobs:
           args: '-no-fail -fmt sarif -out results.sarif ./...'
       - name: Upload SARIF file
         if: ${{ github.actor != 'dependabot[bot]' }}
-        uses: github/codeql-action/upload-sarif@v4
+        uses: github/codeql-action/upload-sarif@v3
         with:
           # Path to SARIF file relative to the root of the repository
           sarif_file: results.sarif

.github/workflows/stalebot.yml

@@ -10,7 +10,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v9
+      - uses: actions/stale@3a9db7e6a41a89f618792c92c0e97cc736e1b13f # v9
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
           stale-pr-message: 'This PR is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 10 days.'

.github/workflows/test-extra.yml

@@ -19,7 +19,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   steps:
   #     - name: Clone
-  #       uses: actions/checkout@v6
+  #       uses: actions/checkout@v5
   #       with:
   #         submodules: true
   #     - name: Dependencies
@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Dependencies
@@ -61,7 +61,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Dependencies
@@ -83,7 +83,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Dependencies
@@ -104,7 +104,7 @@ jobs:
   #  runs-on: ubuntu-latest
   #  steps:
   #    - name: Clone
-  #      uses: actions/checkout@v6
+  #      uses: actions/checkout@v5
   #      with:
   #        submodules: true
   #    - name: Dependencies
@@ -124,7 +124,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   steps:
   #     - name: Clone
-  #       uses: actions/checkout@v6
+  #       uses: actions/checkout@v5
   #       with:
   #         submodules: true
   #     - name: Dependencies
@@ -186,7 +186,7 @@ jobs:
   #           sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
   #           df -h
   #     - name: Clone
-  #       uses: actions/checkout@v6
+  #       uses: actions/checkout@v5
   #       with:
   #         submodules: true
   #     - name: Dependencies
@@ -211,7 +211,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   steps:
   #     - name: Clone
-  #       uses: actions/checkout@v6
+  #       uses: actions/checkout@v5
   #       with:
   #         submodules: true
   #     - name: Dependencies
@@ -232,7 +232,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Dependencies

.github/workflows/test.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: ['1.25.x']
+        go-version: ['1.21.x']
     steps:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
@@ -70,7 +70,7 @@ jobs:
           sudo rm -rfv build || true
           df -h
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Setup Go ${{ matrix.go-version }}
@@ -109,6 +109,11 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y cuda-nvcc-${CUDA_VERSION} libcublas-dev-${CUDA_VERSION}
           export CUDACXX=/usr/local/cuda/bin/nvcc
+
+
+          # The python3-grpc-tools package in 22.04 is too old
+          pip install --user grpcio-tools==1.71.0 grpcio==1.71.0
+
           make -C backend/python/transformers
 
           make backends/huggingface backends/llama-cpp backends/local-store backends/silero-vad backends/piper backends/whisper backends/stablediffusion-ggml
@@ -119,7 +124,7 @@ jobs:
           PATH="$PATH:/root/go/bin" GO_TAGS="tts" make --jobs 5 --output-sync=target test
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3.23
+        uses: mxschmitt/action-tmate@v3.22
         with:
           detached: true
           connect-timeout-seconds: 180
@@ -161,7 +166,7 @@ jobs:
           sudo rm -rfv build || true
           df -h
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Dependencies
@@ -178,20 +183,20 @@ jobs:
             PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-aio e2e-aio
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3.23
+        uses: mxschmitt/action-tmate@v3.22
         with:
           detached: true
           connect-timeout-seconds: 180
           limit-access-to-actor: true
 
   tests-apple:
-    runs-on: macos-latest
+    runs-on: macOS-14
     strategy:
       matrix:
-        go-version: ['1.25.x']
+        go-version: ['1.21.x']
     steps:
       - name: Clone
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           submodules: true
       - name: Setup Go ${{ matrix.go-version }}
@@ -205,7 +210,7 @@ jobs:
       - name: Dependencies
         run: |
           brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm
-          pip install --user --no-cache-dir grpcio-tools grpcio
+          pip install --user --no-cache-dir grpcio-tools==1.71.0 grpcio==1.71.0
       - name: Build llama-cpp-darwin
         run: |
           make protogen-go
@@ -221,7 +226,7 @@ jobs:
           PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3.23
+        uses: mxschmitt/action-tmate@v3.22
         with:
           detached: true
           connect-timeout-seconds: 180

.github/workflows/update_swagger.yaml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v5
       - uses: actions/setup-go@v5
         with:
           go-version: 'stable'
@@ -25,7 +25,7 @@ jobs:
         run: |
           make protogen-go swagger
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v8
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.UPDATE_BOT_TOKEN }}
           push-to-fork: ci-forks/LocalAI

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "docs/themes/hugo-theme-relearn"]
 	path = docs/themes/hugo-theme-relearn
 	url = https://github.com/McShelby/hugo-theme-relearn.git
+[submodule "docs/themes/lotusdocs"]
+	path = docs/themes/lotusdocs
+	url = https://github.com/colinwilson/lotusdocs

.goreleaser.yaml

@@ -22,9 +22,6 @@ builds:
     goarch:
       - amd64
       - arm64
-    ignore:
-      - goos: darwin
-        goarch: amd64
 archives:
   - formats: [ 'binary' ] # this removes the tar of the archives, leaving the binaries alone
     name_template: local-ai-{{ .Tag }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}

AGENTS.md

@@ -1,79 +0,0 @@
-# Build and testing
-
-Building and testing the project depends on the components involved and the platform where development is taking place. Due to the amount of context required it's usually best not to try building or testing the project unless the user requests it. If you must build the project then inspect the Makefile in the project root and the Makefiles of any backends that are effected by changes you are making. In addition the workflows in .github/workflows can be used as a reference when it is unclear how to build or test a component. The primary Makefile contains targets for building inside or outside Docker, if the user has not previously specified a preference then ask which they would like to use.
-
-# Coding style
-
-- The project has the following .editorconfig
-
-```
-root = true
-
-[*]
-indent_style = space
-indent_size = 2
-end_of_line = lf
-charset = utf-8
-trim_trailing_whitespace = true
-insert_final_newline = true
-
-[*.go]
-indent_style = tab
-
-[Makefile]
-indent_style = tab
-
-[*.proto]
-indent_size = 2
-
-[*.py]
-indent_size = 4
-
-[*.js]
-indent_size = 2
-
-[*.yaml]
-indent_size = 2
-
-[*.md]
-trim_trailing_whitespace = false
-```
-
-- Use comments sparingly to explain why code does something, not what it does. Comments are there to add context that would be difficult to deduce from reading the code.
-- Prefer modern Go e.g. use `any` not `interface{}`
-
-# Logging
-
-Use `github.com/mudler/xlog` for logging which has the same API as slog.
-
-# llama.cpp Backend
-
-The llama.cpp backend (`backend/cpp/llama-cpp/grpc-server.cpp`) is a gRPC adaptation of the upstream HTTP server (`llama.cpp/tools/server/server.cpp`). It uses the same underlying server infrastructure from `llama.cpp/tools/server/server-context.cpp`.
-
-## Building and Testing
-
-- Test llama.cpp backend compilation: `make backends/llama-cpp`
-- The backend is built as part of the main build process
-- Check `backend/cpp/llama-cpp/Makefile` for build configuration
-
-## Architecture
-
-- **grpc-server.cpp**: gRPC server implementation, adapts HTTP server patterns to gRPC
-- Uses shared server infrastructure: `server-context.cpp`, `server-task.cpp`, `server-queue.cpp`, `server-common.cpp`
-- The gRPC server mirrors the HTTP server's functionality but uses gRPC instead of HTTP
-
-## Common Issues When Updating llama.cpp
-
-When fixing compilation errors after upstream changes:
-1. Check how `server.cpp` (HTTP server) handles the same change
-2. Look for new public APIs or getter methods
-3. Store copies of needed data instead of accessing private members
-4. Update function calls to match new signatures
-5. Test with `make backends/llama-cpp`
-
-## Key Differences from HTTP Server
-
-- gRPC uses `BackendServiceImpl` class with gRPC service methods
-- HTTP server uses `server_routes` with HTTP handlers
-- Both use the same `server_context` and task queue infrastructure
-- gRPC methods: `LoadModel`, `Predict`, `PredictStream`, `Embedding`, `Rerank`, `TokenizeString`, `GetMetrics`, `Health`

CONTRIBUTING.md

@@ -30,7 +30,6 @@ Thank you for your interest in contributing to LocalAI! We appreciate your time
 3. Install the required dependencies ( see https://localai.io/basics/build/#build-localai-locally )
 4. Build LocalAI: `make build`
 5. Run LocalAI: `./local-ai`
-6. To Build and live reload: `make build-dev`
 
 ## Contributing
 
@@ -77,7 +76,7 @@ LOCALAI_IMAGE_TAG=test LOCALAI_IMAGE=local-ai-aio make run-e2e-aio
 ## Documentation
 
 We are welcome the contribution of the documents, please open new PR or create a new issue. The documentation is available under `docs/` https://github.com/mudler/LocalAI/tree/master/docs
-
+ 
 ## Community and Communication
 
 - You can reach out via the Github issue tracker.

Dockerfile

@@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         ca-certificates curl wget espeak-ng libgomp1 \
-        ffmpeg && \
+        ffmpeg libopenblas-base libopenblas-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -23,7 +23,6 @@ ARG SKIP_DRIVERS=false
 ARG TARGETARCH
 ARG TARGETVARIANT
 ENV BUILD_TYPE=${BUILD_TYPE}
-ARG UBUNTU_VERSION=2204
 
 RUN mkdir -p /run/localai
 RUN echo "default" > /run/localai/capability
@@ -33,33 +32,41 @@ RUN <<EOT bash
     if [ "${BUILD_TYPE}" = "vulkan" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
-            software-properties-common pciutils wget gpg-agent && \
-        wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
-        wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
-        apt-get update && \
-        apt-get install -y \
-            vulkan-sdk && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/* && \
-        echo "vulkan" > /run/localai/capability
+            software-properties-common pciutils sudo wget gpg-agent curl xz-utils && \
+            echo "vulkan" > /run/localai/capability && \
+        if [ "amd64" = "$TARGETARCH" ]; then
+            wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
+            wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
+            apt-get update && \
+            apt-get install -y \
+                vulkan-sdk && \
+            apt-get clean && \
+            rm -rf /var/lib/apt/lists/*
+        fi
+        if [ "arm64" = "$TARGETARCH" ]; then
+            # For ARM64, we need to build the Vulkan SDK manually as there are no packages available
+            mkdir vulkan && cd vulkan && curl -L -o vulkan-sdk.tar.xz https://github.com/mudler/vulkan-sdk-arm/releases/download/1.4.321.1/vulkansdk-ubuntu-22.04-arm-1.4.321.1.tar.xz && \
+            tar -xvf vulkan-sdk.tar.xz && \
+            rm vulkan-sdk.tar.xz && \
+            cd * && \
+            cp -rfv aarch64/* /usr/ && \
+            cd ../.. && \
+            rm -rf vulkan
+        fi
     fi
 EOT
 
 # CuBLAS requirements
 RUN <<EOT bash
-    if ( [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "l4t" ] ) && [ "${SKIP_DRIVERS}" = "false" ]; then
+    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
             software-properties-common pciutils
         if [ "amd64" = "$TARGETARCH" ]; then
-            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/cuda-keyring_1.1-1_all.deb
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
         fi
         if [ "arm64" = "$TARGETARCH" ]; then
-            if [ "${CUDA_MAJOR_VERSION}" = "13" ]; then
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/sbsa/cuda-keyring_1.1-1_all.deb
-            else
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/arm64/cuda-keyring_1.1-1_all.deb
-            fi
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb
         fi
         dpkg -i cuda-keyring_1.1-1_all.deb && \
         rm -f cuda-keyring_1.1-1_all.deb && \
@@ -70,34 +77,26 @@ RUN <<EOT bash
             libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcusparse-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        if [ "${CUDA_MAJOR_VERSION}" = "13" ] && [ "arm64" = "$TARGETARCH" ]; then
-            apt-get install -y --no-install-recommends \
-            libcufile-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libcudnn9-cuda-${CUDA_MAJOR_VERSION} cuda-cupti-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libnvjitlink-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        fi
+            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/* && \
-        echo "nvidia-cuda-${CUDA_MAJOR_VERSION}" > /run/localai/capability
+        echo "nvidia" > /run/localai/capability
     fi
 EOT
 
 RUN <<EOT bash
     if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
-        echo "nvidia-l4t-cuda-${CUDA_MAJOR_VERSION}" > /run/localai/capability
+        echo "nvidia-l4t" > /run/localai/capability
     fi
 EOT
 
 # https://github.com/NVIDIA/Isaac-GR00T/issues/343
 RUN <<EOT bash
     if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
-        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        dpkg -i cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        cp /var/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get -y install cudss cudss-cuda-${CUDA_MAJOR_VERSION} && \
-        wget https://developer.download.nvidia.com/compute/nvpl/25.5/local_installers/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        dpkg -i nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        cp /var/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5/nvpl-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get install -y nvpl
+        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu2204-0.6.0_0.6.0-1_arm64.deb && \
+        dpkg -i cudss-local-tegra-repo-ubuntu2204-0.6.0_0.6.0-1_arm64.deb && \
+        cp /var/cudss-local-tegra-repo-ubuntu2204-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
+        apt-get update && apt-get -y install cudss
     fi
 EOT
 
@@ -184,6 +183,14 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 && \
 COPY --chmod=644 custom-ca-certs/* /usr/local/share/ca-certificates/
 RUN update-ca-certificates
 
+
+# OpenBLAS requirements and stable diffusion
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        libopenblas-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN test -n "$TARGETARCH" \
     || (echo 'warn: missing $TARGETARCH, either set this `ARG` manually, or run using `docker buildkit`')
 
@@ -337,6 +344,6 @@ RUN mkdir -p /models /backends
 HEALTHCHECK --interval=1m --timeout=10m --retries=10 \
   CMD curl -f ${HEALTHCHECK_ENDPOINT} || exit 1
 
-VOLUME /models /backends /configuration
+VOLUME /models /backends
 EXPOSE 8080
 ENTRYPOINT [ "/entrypoint.sh" ]

Makefile

@@ -4,9 +4,6 @@ GOVET=$(GOCMD) vet
 BINARY_NAME=local-ai
 LAUNCHER_BINARY_NAME=local-ai-launcher
 
-CUDA_MAJOR_VERSION?=13
-CUDA_MINOR_VERSION?=0
-
 GORELEASER?=
 
 export BUILD_TYPE?=
@@ -106,10 +103,6 @@ build-launcher: ## Build the launcher application
 
 build-all: build build-launcher ## Build both server and launcher
 
-build-dev: ## Run LocalAI in dev mode with live reload
-	@command -v air >/dev/null 2>&1 || go install github.com/air-verse/air@latest
-	air -c .air.toml
-
 dev-dist:
 	$(GORELEASER) build --snapshot --clean
 
@@ -268,7 +261,7 @@ protoc:
 	  echo "Unsupported OS: $$OS_NAME"; exit 1; \
 	fi; \
 	URL=https://github.com/protocolbuffers/protobuf/releases/download/v31.1/$$FILE; \
-	curl -L $$URL -o protoc.zip && \
+	curl -L -s $$URL -o protoc.zip && \
 	unzip -j -d $(CURDIR) protoc.zip bin/protoc && rm protoc.zip
 
 .PHONY: protogen-go
@@ -287,14 +280,12 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/diffusers
 	$(MAKE) -C backend/python/chatterbox
 	$(MAKE) -C backend/python/vllm
-	$(MAKE) -C backend/python/vibevoice
 
 test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/transformers test
 	$(MAKE) -C backend/python/diffusers test
 	$(MAKE) -C backend/python/chatterbox test
 	$(MAKE) -C backend/python/vllm test
-	$(MAKE) -C backend/python/vibevoice test
 
 DOCKER_IMAGE?=local-ai
 DOCKER_AIO_IMAGE?=local-ai-aio
@@ -385,15 +376,6 @@ backends/llama-cpp-darwin: build
 	bash ./scripts/build/llama-cpp-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)"
 
-backends/neutts: docker-build-neutts docker-save-neutts build
-	./local-ai backends install "ocifile://$(abspath ./backend-images/neutts.tar)"
-
-backends/vllm: docker-build-vllm docker-save-vllm build
-	./local-ai backends install "ocifile://$(abspath ./backend-images/vllm.tar)"
-
-backends/vibevoice: docker-build-vibevoice docker-save-vibevoice build
-	./local-ai backends install "ocifile://$(abspath ./backend-images/vibevoice.tar)"
-
 build-darwin-python-backend: build
 	bash ./scripts/build/python-darwin.sh
 
@@ -450,24 +432,9 @@ docker-save-kitten-tts: backend-images
 docker-save-chatterbox: backend-images
 	docker save local-ai-backend:chatterbox -o backend-images/chatterbox.tar
 
-docker-save-vibevoice: backend-images
-	docker save local-ai-backend:vibevoice -o backend-images/vibevoice.tar
-
-docker-build-neutts:
-	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:neutts -f backend/Dockerfile.python --build-arg BACKEND=neutts ./backend
-
-docker-save-neutts: backend-images
-	docker save local-ai-backend:neutts -o backend-images/neutts.tar
-
 docker-build-kokoro:
 	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:kokoro -f backend/Dockerfile.python --build-arg BACKEND=kokoro ./backend
 
-docker-build-vllm:
-	docker build --build-arg CUDA_MAJOR_VERSION=$(CUDA_MAJOR_VERSION) --build-arg CUDA_MINOR_VERSION=$(CUDA_MINOR_VERSION) --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:vllm -f backend/Dockerfile.python --build-arg BACKEND=vllm ./backend
-
-docker-save-vllm: backend-images
-	docker save local-ai-backend:vllm -o backend-images/vllm.tar
-
 docker-save-kokoro: backend-images
 	docker save local-ai-backend:kokoro -o backend-images/kokoro.tar
 
@@ -496,7 +463,7 @@ docker-save-bark-cpp: backend-images
 	docker save local-ai-backend:bark-cpp -o backend-images/bark-cpp.tar
 
 docker-build-stablediffusion-ggml:
-	docker build --progress=plain --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg CUDA_MAJOR_VERSION=$(CUDA_MAJOR_VERSION) --build-arg CUDA_MINOR_VERSION=$(CUDA_MINOR_VERSION) -t local-ai-backend:stablediffusion-ggml -f backend/Dockerfile.golang --build-arg BACKEND=stablediffusion-ggml .
+	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:stablediffusion-ggml -f backend/Dockerfile.golang --build-arg BACKEND=stablediffusion-ggml .
 
 docker-save-stablediffusion-ggml: backend-images
 	docker save local-ai-backend:stablediffusion-ggml -o backend-images/stablediffusion-ggml.tar
@@ -504,6 +471,9 @@ docker-save-stablediffusion-ggml: backend-images
 docker-build-rerankers:
 	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:rerankers -f backend/Dockerfile.python --build-arg BACKEND=rerankers .
 
+docker-build-vllm:
+	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:vllm -f backend/Dockerfile.python --build-arg BACKEND=vllm .
+
 docker-build-transformers:
 	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:transformers -f backend/Dockerfile.python --build-arg BACKEND=transformers .
 
@@ -514,7 +484,7 @@ docker-save-diffusers: backend-images
 	docker save local-ai-backend:diffusers -o backend-images/diffusers.tar
 
 docker-build-whisper:
-	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg CUDA_MAJOR_VERSION=$(CUDA_MAJOR_VERSION) --build-arg CUDA_MINOR_VERSION=$(CUDA_MINOR_VERSION) -t local-ai-backend:whisper -f backend/Dockerfile.golang --build-arg BACKEND=whisper  .
+	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:whisper -f backend/Dockerfile.golang --build-arg BACKEND=whisper  .
 
 docker-save-whisper: backend-images
 	docker save local-ai-backend:whisper -o backend-images/whisper.tar
@@ -531,13 +501,10 @@ docker-build-bark:
 docker-build-chatterbox:
 	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:chatterbox -f backend/Dockerfile.python --build-arg BACKEND=chatterbox ./backend
 
-docker-build-vibevoice:
-	docker build --progress=plain --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:vibevoice -f backend/Dockerfile.python --build-arg BACKEND=vibevoice ./backend
-
 docker-build-exllama2:
 	docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:exllama2 -f backend/Dockerfile.python --build-arg BACKEND=exllama2 .
 
-docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-transformers docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-bark docker-build-chatterbox docker-build-vibevoice docker-build-exllama2
+docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-transformers docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-bark docker-build-chatterbox docker-build-exllama2
 
 ########################################################
 ### END Backends

README.md

@@ -33,7 +33,7 @@
 <img src="https://img.shields.io/badge/X-%23000000.svg?style=for-the-badge&logo=X&logoColor=white&label=LocalAI_API" alt="Follow LocalAI_API"/>
 </a>
 <a href="https://discord.gg/uJAeKSAGDy" target="blank">
-<img src="https://img.shields.io/badge/dynamic/json?color=blue&label=Discord&style=for-the-badge&query=approximate_member_count&url=https%3A%2F%2Fdiscordapp.com%2Fapi%2Finvites%2FuJAeKSAGDy%3Fwith_counts%3Dtrue&logo=discord" alt="Join LocalAI Discord Community"/>
+<img src="https://dcbadge.vercel.app/api/server/uJAeKSAGDy?style=flat-square&theme=default-inverted" alt="Join LocalAI Discord Community"/>
 </a>
 </p>
 
@@ -43,7 +43,7 @@
 
 > :bulb: Get help - [❓FAQ](https://localai.io/faq/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [:speech_balloon: Discord](https://discord.gg/uJAeKSAGDy) [:book: Documentation website](https://localai.io/)
 >
-> [💻 Quickstart](https://localai.io/basics/getting_started/) [🖼️ Models](https://models.localai.io/) [🚀 Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap) [🛫 Examples](https://github.com/mudler/LocalAI-examples) Try on 
+> [💻 Quickstart](https://localai.io/basics/getting_started/) [🖼️ Models](https://models.localai.io/) [🚀 Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap) [🌍 Explorer](https://explorer.localai.io) [🛫 Examples](https://github.com/mudler/LocalAI-examples) Try on 
 [![Telegram](https://img.shields.io/badge/Telegram-2CA5E0?style=for-the-badge&logo=telegram&logoColor=white)](https://t.me/localaiofficial_bot)
 
 [![tests](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml)[![Build and Release](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml)[![build container images](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml)[![Bump dependencies](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml)[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/localai)](https://artifacthub.io/packages/search?repo=localai)
@@ -80,18 +80,8 @@
   </tr>
 </table>
 
-## Screenshots / Video
+## Screenshots
 
-### Youtube video
-
-<h1 align="center">
-  <br>
-  <a href="https://www.youtube.com/watch?v=PDqYhB9nNHA" target="_blank"> <img width="300" src="https://img.youtube.com/vi/PDqYhB9nNHA/0.jpg"> </a><br>
-<br>
-</h1>
-
-
-### Screenshots
 
 | Talk Interface | Generate Audio |
 | --- | --- |
@@ -118,7 +108,7 @@ Run the installer script:
 curl https://localai.io/install.sh | sh
 ```
 
-For more installation options, see [Installer Options](https://localai.io/installation/).
+For more installation options, see [Installer Options](https://localai.io/docs/advanced/installer/).
 
 ### macOS Download:
 
@@ -126,17 +116,8 @@ For more installation options, see [Installer Options](https://localai.io/instal
   <img src="https://img.shields.io/badge/Download-macOS-blue?style=for-the-badge&logo=apple&logoColor=white" alt="Download LocalAI for macOS"/>
 </a>
 
-> Note: the DMGs are not signed by Apple as quarantined. See https://github.com/mudler/LocalAI/issues/6268 for a workaround, fix is tracked here: https://github.com/mudler/LocalAI/issues/6244
-
 Or run with docker:
 
-> **💡 Docker Run vs Docker Start**
-> 
-> - `docker run` creates and starts a new container. If a container with the same name already exists, this command will fail.
-> - `docker start` starts an existing container that was previously created with `docker run`.
-> 
-> If you've already run LocalAI before and want to start it again, use: `docker start -i local-ai`
-
 ### CPU only image:
 
 ```bash
@@ -146,9 +127,6 @@ docker run -ti --name local-ai -p 8080:8080 localai/localai:latest
 ### NVIDIA GPU Images:
 
 ```bash
-# CUDA 13.0
-docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-13
-
 # CUDA 12.0
 docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12
 
@@ -156,11 +134,7 @@ docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gp
 docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-11
 
 # NVIDIA Jetson (L4T) ARM64
-# CUDA 12 (for Nvidia AGX Orin and similar platforms)
 docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-nvidia-l4t-arm64
-
-# CUDA 13 (for Nvidia DGX Spark)
-docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-nvidia-l4t-arm64-cuda-13
 ```
 
 ### AMD GPU Images (ROCm):
@@ -187,9 +161,6 @@ docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-vulkan
 # CPU version
 docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-cpu
 
-# NVIDIA CUDA 13 version
-docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-nvidia-cuda-13
-
 # NVIDIA CUDA 12 version
 docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-nvidia-cuda-12
 
@@ -222,14 +193,10 @@ local-ai run oci://localai/phi-2:latest
 
 > ⚡ **Automatic Backend Detection**: When you install models from the gallery or YAML files, LocalAI automatically detects your system's GPU capabilities (NVIDIA, AMD, Intel) and downloads the appropriate backend. For advanced configuration options, see [GPU Acceleration](https://localai.io/features/gpu-acceleration/#automatic-backend-detection).
 
-For more information, see [💻 Getting started](https://localai.io/basics/getting_started/index.html), if you are interested in our roadmap items and future enhancements, you can see the [Issues labeled as Roadmap here](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
+For more information, see [💻 Getting started](https://localai.io/basics/getting_started/index.html)
 
 ## 📰 Latest project news
 
-- December 2025: [Dynamic Memory Resource reclaimer](https://github.com/mudler/LocalAI/pull/7583), [Automatic fitting of models to multiple GPUS(llama.cpp)](https://github.com/mudler/LocalAI/pull/7584), [Added Vibevoice backend](https://github.com/mudler/LocalAI/pull/7494)
-- November 2025: Major improvements to the UX. Among these: [Import models via URL](https://github.com/mudler/LocalAI/pull/7245) and [Multiple chats and history](https://github.com/mudler/LocalAI/pull/7325)
-- October 2025: 🔌 [Model Context Protocol (MCP)](https://localai.io/docs/features/mcp/) support added for agentic capabilities with external tools
-- September 2025: New Launcher application for MacOS and Linux, extended support to many backends for Mac and Nvidia L4T devices. Models: Added MLX-Audio, WAN 2.2. WebUI improvements and Python-based backends now ships portable python environments.
 - August 2025: MLX, MLX-VLM, Diffusers and llama.cpp are now supported on Mac M1/M2/M3+ chips ( with `development` suffix in the gallery ): https://github.com/mudler/LocalAI/pull/6049 https://github.com/mudler/LocalAI/pull/6119 https://github.com/mudler/LocalAI/pull/6121 https://github.com/mudler/LocalAI/pull/6060
 - July/August 2025: 🔍 [Object Detection](https://localai.io/features/object-detection/) added to the API featuring [rf-detr](https://github.com/roboflow/rf-detr)
 - July 2025: All backends migrated outside of the main binary. LocalAI is now more lightweight, small, and automatically downloads the required backend to run the model. [Read the release notes](https://github.com/mudler/LocalAI/releases/tag/v3.2.0)
@@ -268,7 +235,7 @@ Roadmap items: [List of issues](https://github.com/mudler/LocalAI/issues?q=is%3A
 - 🔍 [Object Detection](https://localai.io/features/object-detection/)
 - 📈 [Reranker API](https://localai.io/features/reranker/)
 - 🆕🖧 [P2P Inferencing](https://localai.io/features/distribute/)
-- 🆕🔌 [Model Context Protocol (MCP)](https://localai.io/docs/features/mcp/) - Agentic capabilities with external tools and [LocalAGI's Agentic capabilities](https://github.com/mudler/LocalAGI)
+- [Agentic capabilities](https://github.com/mudler/LocalAGI)
 - 🔊 Voice activity detection (Silero-VAD support)
 - 🌍 Integrated WebUI!
 
@@ -279,40 +246,38 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration
 ### Text Generation & Language Models
 | Backend | Description | Acceleration Support |
 |---------|-------------|---------------------|
-| **llama.cpp** | LLM inference in C/C++ | CUDA 11/12/13, ROCm, Intel SYCL, Vulkan, Metal, CPU |
-| **vLLM** | Fast LLM inference with PagedAttention | CUDA 12/13, ROCm, Intel |
-| **transformers** | HuggingFace transformers framework | CUDA 11/12/13, ROCm, Intel, CPU |
-| **exllama2** | GPTQ inference library | CUDA 12/13 |
+| **llama.cpp** | LLM inference in C/C++ | CUDA 11/12, ROCm, Intel SYCL, Vulkan, Metal, CPU |
+| **vLLM** | Fast LLM inference with PagedAttention | CUDA 12, ROCm, Intel |
+| **transformers** | HuggingFace transformers framework | CUDA 11/12, ROCm, Intel, CPU |
+| **exllama2** | GPTQ inference library | CUDA 12 |
 | **MLX** | Apple Silicon LLM inference | Metal (M1/M2/M3+) |
 | **MLX-VLM** | Apple Silicon Vision-Language Models | Metal (M1/M2/M3+) |
 
 ### Audio & Speech Processing
 | Backend | Description | Acceleration Support |
 |---------|-------------|---------------------|
-| **whisper.cpp** | OpenAI Whisper in C/C++ | CUDA 12/13, ROCm, Intel SYCL, Vulkan, CPU |
-| **faster-whisper** | Fast Whisper with CTranslate2 | CUDA 12/13, ROCm, Intel, CPU |
-| **bark** | Text-to-audio generation | CUDA 12/13, ROCm, Intel |
+| **whisper.cpp** | OpenAI Whisper in C/C++ | CUDA 12, ROCm, Intel SYCL, Vulkan, CPU |
+| **faster-whisper** | Fast Whisper with CTranslate2 | CUDA 12, ROCm, Intel, CPU |
+| **bark** | Text-to-audio generation | CUDA 12, ROCm, Intel |
 | **bark-cpp** | C++ implementation of Bark | CUDA, Metal, CPU |
-| **coqui** | Advanced TTS with 1100+ languages | CUDA 12/13, ROCm, Intel, CPU |
-| **kokoro** | Lightweight TTS model | CUDA 12/13, ROCm, Intel, CPU |
-| **chatterbox** | Production-grade TTS | CUDA 11/12/13, CPU |
+| **coqui** | Advanced TTS with 1100+ languages | CUDA 12, ROCm, Intel, CPU |
+| **kokoro** | Lightweight TTS model | CUDA 12, ROCm, Intel, CPU |
+| **chatterbox** | Production-grade TTS | CUDA 11/12, CPU |
 | **piper** | Fast neural TTS system | CPU |
 | **kitten-tts** | Kitten TTS models | CPU |
 | **silero-vad** | Voice Activity Detection | CPU |
-| **neutts** | Text-to-speech with voice cloning | CUDA 12/13, ROCm, CPU |
-| **vibevoice** | Real-time TTS with voice cloning | CUDA 12/13, ROCm, Intel, CPU |
 
 ### Image & Video Generation
 | Backend | Description | Acceleration Support |
 |---------|-------------|---------------------|
-| **stablediffusion.cpp** | Stable Diffusion in C/C++ | CUDA 12/13, Intel SYCL, Vulkan, CPU |
-| **diffusers** | HuggingFace diffusion models | CUDA 11/12/13, ROCm, Intel, Metal, CPU |
+| **stablediffusion.cpp** | Stable Diffusion in C/C++ | CUDA 12, Intel SYCL, Vulkan, CPU |
+| **diffusers** | HuggingFace diffusion models | CUDA 11/12, ROCm, Intel, Metal, CPU |
 
 ### Specialized AI Tasks
 | Backend | Description | Acceleration Support |
 |---------|-------------|---------------------|
-| **rfdetr** | Real-time object detection | CUDA 12/13, Intel, CPU |
-| **rerankers** | Document reranking API | CUDA 11/12/13, ROCm, Intel, CPU |
+| **rfdetr** | Real-time object detection | CUDA 12, Intel, CPU |
+| **rerankers** | Document reranking API | CUDA 11/12, ROCm, Intel, CPU |
 | **local-store** | Vector database | CPU |
 | **huggingface** | HuggingFace API integration | API-based |
 
@@ -322,13 +287,11 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration
 |-------------------|-------------------|------------------|
 | **NVIDIA CUDA 11** | llama.cpp, whisper, stablediffusion, diffusers, rerankers, bark, chatterbox | Nvidia hardware |
 | **NVIDIA CUDA 12** | All CUDA-compatible backends | Nvidia hardware |
-| **NVIDIA CUDA 13** | All CUDA-compatible backends | Nvidia hardware |
-| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark, neutts, vibevoice | AMD Graphics |
-| **Intel oneAPI** | llama.cpp, whisper, stablediffusion, vllm, transformers, diffusers, rfdetr, rerankers, exllama2, coqui, kokoro, bark, vibevoice | Intel Arc, Intel iGPUs |
+| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark | AMD Graphics |
+| **Intel oneAPI** | llama.cpp, whisper, stablediffusion, vllm, transformers, diffusers, rfdetr, rerankers, exllama2, coqui, kokoro, bark | Intel Arc, Intel iGPUs |
 | **Apple Metal** | llama.cpp, whisper, diffusers, MLX, MLX-VLM, bark-cpp | Apple M1/M2/M3+ |
 | **Vulkan** | llama.cpp, whisper, stablediffusion | Cross-platform GPUs |
-| **NVIDIA Jetson (CUDA 12)** | llama.cpp, whisper, stablediffusion, diffusers, rfdetr | ARM64 embedded AI (AGX Orin, etc.) |
-| **NVIDIA Jetson (CUDA 13)** | llama.cpp, whisper, stablediffusion, diffusers, rfdetr | ARM64 embedded AI (DGX Spark) |
+| **NVIDIA Jetson** | llama.cpp, whisper, stablediffusion, diffusers, rfdetr | ARM64 embedded AI |
 | **CPU Optimized** | All backends | AVX/AVX2/AVX512, quantization support |
 
 ### 🔗 Community and integrations
@@ -341,12 +304,6 @@ WebUIs:
 - https://github.com/go-skynet/LocalAI-frontend
 - QA-Pilot(An interactive chat project that leverages LocalAI LLMs for rapid understanding and navigation of GitHub code repository) https://github.com/reid41/QA-Pilot
 
-Agentic Libraries:
-- https://github.com/mudler/cogito
-
-MCPs:
-- https://github.com/mudler/MCPs
-
 Model galleries
 - https://github.com/go-skynet/model-gallery
 
@@ -421,10 +378,6 @@ A huge thank you to our generous sponsors who support this project covering CI e
   </a>
 </p>
 
-### Individual sponsors
-
-A special thanks to individual sponsors that contributed to the project, a full list is in [Github](https://github.com/sponsors/mudler) and [buymeacoffee](https://buymeacoffee.com/mudler), a special shout out goes to [drikster80](https://github.com/drikster80) for being generous. Thank you everyone!
-
 ## 🌟 Star history
 
 [![LocalAI Star history Chart](https://api.star-history.com/svg?repos=go-skynet/LocalAI&type=Date)](https://star-history.com/#go-skynet/LocalAI&Date)

backend/Dockerfile.golang

@@ -13,14 +13,13 @@ ENV DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 ARG TARGETVARIANT
 ARG GO_VERSION=1.22.6
-ARG UBUNTU_VERSION=2204
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         build-essential \
         git ccache \
         ca-certificates \
-        make cmake wget \
+        make cmake \
         curl unzip \
         libssl-dev && \
     apt-get clean && \
@@ -33,38 +32,46 @@ ENV PATH=/usr/local/cuda/bin:${PATH}
 # HipBLAS requirements
 ENV PATH=/opt/rocm/bin:${PATH}
 
-
 # Vulkan requirements
 RUN <<EOT bash
     if [ "${BUILD_TYPE}" = "vulkan" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
-            software-properties-common pciutils wget gpg-agent && \
-        wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
-        wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
-        apt-get update && \
-        apt-get install -y \
-            vulkan-sdk && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
+            software-properties-common pciutils sudo wget gpg-agent curl xz-utils && \
+            echo "vulkan" > /run/localai/capability && \
+        if [ "amd64" = "$TARGETARCH" ]; then
+            wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
+            wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
+            apt-get update && \
+            apt-get install -y \
+                vulkan-sdk && \
+            apt-get clean && \
+            rm -rf /var/lib/apt/lists/*
+        fi
+        if [ "arm64" = "$TARGETARCH" ]; then
+            # For ARM64, we need to build the Vulkan SDK manually as there are no packages available
+            mkdir vulkan && cd vulkan && curl -L -o vulkan-sdk.tar.xz https://github.com/mudler/vulkan-sdk-arm/releases/download/1.4.321.1/vulkansdk-ubuntu-22.04-arm-1.4.321.1.tar.xz && \
+            tar -xvf vulkan-sdk.tar.xz && \
+            rm vulkan-sdk.tar.xz && \
+            cd * && \
+            cp -rfv aarch64/* /usr/ && \
+            cd ../.. && \
+            rm -rf vulkan
+        fi
     fi
 EOT
 
 # CuBLAS requirements
 RUN <<EOT bash
-    if ( [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "l4t" ] ) && [ "${SKIP_DRIVERS}" = "false" ]; then
+    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
             software-properties-common pciutils
         if [ "amd64" = "$TARGETARCH" ]; then
-            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/cuda-keyring_1.1-1_all.deb
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
         fi
         if [ "arm64" = "$TARGETARCH" ]; then
-            if [ "${CUDA_MAJOR_VERSION}" = "13" ]; then
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/sbsa/cuda-keyring_1.1-1_all.deb
-            else
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/arm64/cuda-keyring_1.1-1_all.deb
-            fi
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb
         fi
         dpkg -i cuda-keyring_1.1-1_all.deb && \
         rm -f cuda-keyring_1.1-1_all.deb && \
@@ -75,31 +82,12 @@ RUN <<EOT bash
             libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcusparse-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        if [ "${CUDA_MAJOR_VERSION}" = "13" ] && [ "arm64" = "$TARGETARCH" ]; then
-            apt-get install -y --no-install-recommends \
-            libcufile-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libcudnn9-cuda-${CUDA_MAJOR_VERSION} cuda-cupti-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libnvjitlink-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        fi
+            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/*
     fi
 EOT
 
-
-# https://github.com/NVIDIA/Isaac-GR00T/issues/343
-RUN <<EOT bash
-    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
-        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        dpkg -i cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        cp /var/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get -y install cudss cudss-cuda-${CUDA_MAJOR_VERSION} && \
-        wget https://developer.download.nvidia.com/compute/nvpl/25.5/local_installers/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        dpkg -i nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        cp /var/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5/nvpl-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get install -y nvpl
-    fi
-EOT
-
 # If we are building with clblas support, we need the libraries for the builds
 RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
         apt-get update && \

backend/Dockerfile.llama-cpp

@@ -20,7 +20,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         ca-certificates \
         build-essential curl libssl-dev \
-        git wget && \
+        git && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -62,7 +62,6 @@ ENV DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 ARG TARGETVARIANT
 ARG GO_VERSION=1.22.6
-ARG UBUNTU_VERSION=2204
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -71,7 +70,7 @@ RUN apt-get update && \
         ca-certificates \
         make \
         curl unzip \
-        libssl-dev wget && \
+        libssl-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -81,38 +80,46 @@ ENV PATH=/usr/local/cuda/bin:${PATH}
 # HipBLAS requirements
 ENV PATH=/opt/rocm/bin:${PATH}
 
-
 # Vulkan requirements
 RUN <<EOT bash
     if [ "${BUILD_TYPE}" = "vulkan" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
-            software-properties-common pciutils wget gpg-agent && \
-        wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
-        wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
-        apt-get update && \
-        apt-get install -y \
-            vulkan-sdk && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
+            software-properties-common pciutils sudo wget gpg-agent curl xz-utils libxcb1 libx11-6 && \
+            echo "vulkan" > /run/localai/capability && \
+        if [ "amd64" = "$TARGETARCH" ]; then
+            wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
+            wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
+            apt-get update && \
+            apt-get install -y \
+                vulkan-sdk && \
+            apt-get clean && \
+            rm -rf /var/lib/apt/lists/*
+        fi
+        if [ "arm64" = "$TARGETARCH" ]; then
+            # For ARM64, we need to build the Vulkan SDK manually as there are no packages available
+            mkdir vulkan && cd vulkan && curl -L -o vulkan-sdk.tar.xz https://github.com/mudler/vulkan-sdk-arm/releases/download/1.4.321.1/vulkansdk-ubuntu-22.04-arm-1.4.321.1.tar.xz && \
+            tar -xvf vulkan-sdk.tar.xz && \
+            rm vulkan-sdk.tar.xz && \
+            cd * && \
+            cp -rfv aarch64/* /usr/ && vulkaninfo \
+            cd ../.. && \
+            rm -rf vulkan
+        fi
     fi
 EOT
 
 # CuBLAS requirements
 RUN <<EOT bash
-    if ( [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "l4t" ] ) && [ "${SKIP_DRIVERS}" = "false" ]; then
+    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
             software-properties-common pciutils
         if [ "amd64" = "$TARGETARCH" ]; then
-            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/cuda-keyring_1.1-1_all.deb
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
         fi
         if [ "arm64" = "$TARGETARCH" ]; then
-            if [ "${CUDA_MAJOR_VERSION}" = "13" ]; then
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/sbsa/cuda-keyring_1.1-1_all.deb
-            else
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/arm64/cuda-keyring_1.1-1_all.deb
-            fi
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb
         fi
         dpkg -i cuda-keyring_1.1-1_all.deb && \
         rm -f cuda-keyring_1.1-1_all.deb && \
@@ -123,31 +130,12 @@ RUN <<EOT bash
             libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcusparse-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        if [ "${CUDA_MAJOR_VERSION}" = "13" ] && [ "arm64" = "$TARGETARCH" ]; then
-            apt-get install -y --no-install-recommends \
-            libcufile-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libcudnn9-cuda-${CUDA_MAJOR_VERSION} cuda-cupti-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libnvjitlink-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        fi
+            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/*
     fi
 EOT
 
-
-# https://github.com/NVIDIA/Isaac-GR00T/issues/343
-RUN <<EOT bash
-    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
-        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        dpkg -i cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        cp /var/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get -y install cudss cudss-cuda-${CUDA_MAJOR_VERSION} && \
-        wget https://developer.download.nvidia.com/compute/nvpl/25.5/local_installers/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        dpkg -i nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        cp /var/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5/nvpl-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get install -y nvpl
-    fi
-EOT
-
 # If we are building with clblas support, we need the libraries for the builds
 RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
         apt-get update && \
@@ -222,7 +210,7 @@ EOT
 
 
 # Copy libraries using a script to handle architecture differences
-RUN make -BC /LocalAI/backend/cpp/llama-cpp package
+RUN make -C /LocalAI/backend/cpp/llama-cpp package
 
 
 FROM scratch

backend/Dockerfile.python

@@ -12,7 +12,6 @@ ENV CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION}
 ENV DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG UBUNTU_VERSION=2204
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -22,24 +21,17 @@ RUN apt-get update && \
         espeak-ng \
         curl \
         libssl-dev \
-        git wget \
+        git \
         git-lfs \
         unzip clang \
         upx-ucl \
         curl python3-pip \
         python-is-python3 \
         python3-dev llvm \
-        python3-venv make cmake && \
+        python3-venv make && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN <<EOT bash
-    if [ "${UBUNTU_VERSION}" = "2404" ]; then
-        pip install --break-system-packages --user --upgrade pip
-    else
-        pip install --upgrade pip
-    fi
-EOT
+    rm -rf /var/lib/apt/lists/* && \
+    pip install --upgrade pip
 
 
 # Cuda
@@ -53,32 +45,41 @@ RUN <<EOT bash
     if [ "${BUILD_TYPE}" = "vulkan" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
-            software-properties-common pciutils wget gpg-agent && \
-        wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
-        wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
-        apt-get update && \
-        apt-get install -y \
-            vulkan-sdk && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
+            software-properties-common pciutils sudo wget gpg-agent curl xz-utils && \
+            echo "vulkan" > /run/localai/capability && \
+        if [ "amd64" = "$TARGETARCH" ]; then
+            wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
+            wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
+            apt-get update && \
+            apt-get install -y \
+                vulkan-sdk && \
+            apt-get clean && \
+            rm -rf /var/lib/apt/lists/*
+        fi
+        if [ "arm64" = "$TARGETARCH" ]; then
+            # For ARM64, we need to build the Vulkan SDK manually as there are no packages available
+            mkdir vulkan && cd vulkan && curl -L -o vulkan-sdk.tar.xz https://github.com/mudler/vulkan-sdk-arm/releases/download/1.4.321.1/vulkansdk-ubuntu-22.04-arm-1.4.321.1.tar.xz && \
+            tar -xvf vulkan-sdk.tar.xz && \
+            rm vulkan-sdk.tar.xz && \
+            cd * && \
+            cp -rfv aarch64/* /usr/ && \
+            cd ../.. && \
+            rm -rf vulkan
+        fi
     fi
 EOT
 
 # CuBLAS requirements
 RUN <<EOT bash
-    if ( [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "l4t" ] ) && [ "${SKIP_DRIVERS}" = "false" ]; then
+    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         apt-get update && \
         apt-get install -y  --no-install-recommends \
             software-properties-common pciutils
         if [ "amd64" = "$TARGETARCH" ]; then
-            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/cuda-keyring_1.1-1_all.deb
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
         fi
         if [ "arm64" = "$TARGETARCH" ]; then
-            if [ "${CUDA_MAJOR_VERSION}" = "13" ]; then
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/sbsa/cuda-keyring_1.1-1_all.deb
-            else
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/arm64/cuda-keyring_1.1-1_all.deb
-            fi
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb
         fi
         dpkg -i cuda-keyring_1.1-1_all.deb && \
         rm -f cuda-keyring_1.1-1_all.deb && \
@@ -89,31 +90,12 @@ RUN <<EOT bash
             libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcusparse-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        if [ "${CUDA_MAJOR_VERSION}" = "13" ] && [ "arm64" = "$TARGETARCH" ]; then
-            apt-get install -y --no-install-recommends \
-            libcufile-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libcudnn9-cuda-${CUDA_MAJOR_VERSION} cuda-cupti-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libnvjitlink-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        fi
+            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/*
     fi
 EOT
 
-
-# https://github.com/NVIDIA/Isaac-GR00T/issues/343
-RUN <<EOT bash
-    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
-        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        dpkg -i cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        cp /var/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get -y install cudss cudss-cuda-${CUDA_MAJOR_VERSION} && \
-        wget https://developer.download.nvidia.com/compute/nvpl/25.5/local_installers/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        dpkg -i nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        cp /var/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5/nvpl-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get install -y nvpl
-    fi
-EOT
-
 # If we are building with clblas support, we need the libraries for the builds
 RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
         apt-get update && \
@@ -134,11 +116,6 @@ RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency
         ldconfig \
     ; fi
-
-RUN if [ "${BUILD_TYPE}" = "hipblas" ]; then \
-    ln -s /opt/rocm-**/lib/llvm/lib/libomp.so /usr/lib/libomp.so \
-    ; fi
-
 # Install uv as a system package
 RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/bin sh
 ENV PATH="/root/.cargo/bin:${PATH}"
@@ -146,14 +123,7 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
 # Install grpcio-tools (the version in 22.04 is too old)
-RUN <<EOT bash
-    if [ "${UBUNTU_VERSION}" = "2404" ]; then
-        pip install --break-system-packages --user grpcio-tools==1.71.0 grpcio==1.71.0
-    else
-        pip install grpcio-tools==1.71.0 grpcio==1.71.0
-    fi
-EOT
-
+RUN pip install --user grpcio-tools==1.71.0 grpcio==1.71.0
 
 COPY python/${BACKEND} /${BACKEND}
 COPY backend.proto /${BACKEND}/backend.proto

backend/backend.proto

@@ -154,10 +154,6 @@ message PredictOptions {
   repeated string Videos = 45;
   repeated string Audios = 46;
   string CorrelationId = 47;
-  string Tools = 48;  // JSON array of available tools/functions for tool calling
-  string ToolChoice = 49;  // JSON string or object specifying tool choice behavior
-  int32 Logprobs = 50;  // Number of top logprobs to return (maps to OpenAI logprobs parameter)
-  int32 TopLogprobs = 51;  // Number of top logprobs to return per token (maps to OpenAI top_logprobs parameter)
 }
 
 // The response message containing the result
@@ -168,7 +164,6 @@ message Reply {
   double timing_prompt_processing = 4;
   double timing_token_generation = 5;
   bytes audio = 6;
-  bytes logprobs = 7;  // JSON-encoded logprobs data matching OpenAI format
 }
 
 message GrammarTrigger {
@@ -282,7 +277,6 @@ message TranscriptRequest {
   uint32 threads = 4;
   bool translate = 5;
   bool diarize = 6;
-  string prompt = 7;
 }
 
 message TranscriptResult {
@@ -301,6 +295,7 @@ message TranscriptSegment {
 message GenerateImageRequest {
   int32 height = 1;
   int32 width = 2;
+  int32 mode = 3;
   int32 step = 4;
   int32 seed = 5;
   string positive_prompt = 6;
@@ -387,11 +382,6 @@ message StatusResponse {
 message Message {
   string role = 1;
   string content = 2;
-  // Optional fields for OpenAI-compatible message format
-  string name = 3;                    // Tool name (for tool messages)
-  string tool_call_id = 4;            // Tool call ID (for tool messages)
-  string reasoning_content = 5;       // Reasoning content (for thinking models)
-  string tool_calls = 6;              // Tool calls as JSON string (for assistant messages with tool calls)
 }
 
 message DetectOptions {

backend/cpp/llama-cpp/CMakeLists.txt

@@ -57,7 +57,7 @@ add_library(hw_grpc_proto
   ${hw_proto_srcs}
   ${hw_proto_hdrs} )
 
-add_executable(${TARGET} grpc-server.cpp json.hpp httplib.h)
+add_executable(${TARGET} grpc-server.cpp utils.hpp json.hpp httplib.h)
 
 target_include_directories(${TARGET} PRIVATE ../llava)
 target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR})

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=ced765be44ce173c374f295b3c6f4175f8fd109b
+LLAMA_VERSION?=4807e8f96a61b2adccebd5e57444c94d18de7264
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/patches/01-llava.patch

@@ -0,0 +1,13 @@
+diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
+index 3cd0d2fa..6c5e811a 100644
+--- a/tools/mtmd/clip.cpp
++++ b/tools/mtmd/clip.cpp
+@@ -2608,7 +2608,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
+                 struct ggml_tensor * patches = ggml_graph_get_tensor(gf, "patches");
+                 int* patches_data = (int*)malloc(ggml_nbytes(patches));
+                 for (int i = 0; i < num_patches; i++) {
+-                    patches_data[i] = i + 1;
++                    patches_data[i] = i;
+                 }
+                 ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches));
+                 free(patches_data);

backend/cpp/llama-cpp/prepare.sh

@@ -1,24 +1,18 @@
 #!/bin/bash
 
 ## Patches
-
 ## Apply patches from the `patches` directory
-if [ -d "patches" ]; then
-    for patch in $(ls patches); do
-        echo "Applying patch $patch"
-        patch -d llama.cpp/ -p1 < patches/$patch
-    done 
-fi
+for patch in $(ls patches); do
+    echo "Applying patch $patch"
+    patch -d llama.cpp/ -p1 < patches/$patch
+done 
 
 set -e
 
-for file in $(ls llama.cpp/tools/server/); do
-    cp -rfv llama.cpp/tools/server/$file llama.cpp/tools/grpc-server/
-done
-
 cp -r CMakeLists.txt llama.cpp/tools/grpc-server/
 cp -r grpc-server.cpp llama.cpp/tools/grpc-server/
 cp -rfv llama.cpp/vendor/nlohmann/json.hpp llama.cpp/tools/grpc-server/
+cp -rfv llama.cpp/tools/server/utils.hpp llama.cpp/tools/grpc-server/
 cp -rfv llama.cpp/vendor/cpp-httplib/httplib.h llama.cpp/tools/grpc-server/
 
 set +e
@@ -29,3 +23,30 @@ else
 fi
 set -e
 
+# Now to keep maximum compatibility with the original server.cpp, we need to remove the index.html.gz.hpp and loading.html.hpp includes
+# and remove the main function
+# TODO: upstream this to the original server.cpp by extracting the upstream main function to a separate file
+awk '
+/int[ \t]+main[ \t]*\(/ {          # If the line starts the main function
+    in_main=1;                     # Set a flag
+    open_braces=0;                 # Track number of open braces
+}
+in_main {
+    open_braces += gsub(/\{/, "{"); # Count opening braces
+    open_braces -= gsub(/\}/, "}"); # Count closing braces
+    if (open_braces == 0) {         # If all braces are closed
+        in_main=0;                  # End skipping
+    }
+    next;                           # Skip lines inside main
+}
+!in_main                           # Print lines not inside main
+' "llama.cpp/tools/server/server.cpp" > llama.cpp/tools/grpc-server/server.cpp
+
+# remove index.html.gz.hpp and loading.html.hpp includes
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # macOS
+    sed -i '' '/#include "index\.html\.gz\.hpp"/d; /#include "loading\.html\.hpp"/d' llama.cpp/tools/grpc-server/server.cpp
+else
+    # Linux and others
+    sed -i '/#include "index\.html\.gz\.hpp"/d; /#include "loading\.html\.hpp"/d' llama.cpp/tools/grpc-server/server.cpp
+fi

backend/go/local-store/debug.go

@@ -4,11 +4,11 @@
 package main
 
 import (
-	"github.com/mudler/xlog"
+	"github.com/rs/zerolog/log"
 )
 
 func assert(cond bool, msg string) {
 	if !cond {
-		xlog.Fatal().Stack().Msg(msg)
+		log.Fatal().Stack().Msg(msg)
 	}
 }

backend/go/local-store/main.go

@@ -7,7 +7,8 @@ import (
 	"os"
 
 	grpc "github.com/mudler/LocalAI/pkg/grpc"
-	"github.com/mudler/xlog"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 )
 
 var (
@@ -15,7 +16,7 @@ var (
 )
 
 func main() {
-	xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(os.Getenv("LOCALAI_LOG_LEVEL")), os.Getenv("LOCALAI_LOG_FORMAT")))
+	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
 
 	flag.Parse()
 

backend/go/local-store/store.go

@@ -12,7 +12,7 @@ import (
 	"github.com/mudler/LocalAI/pkg/grpc/base"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 
-	"github.com/mudler/xlog"
+	"github.com/rs/zerolog/log"
 )
 
 type Store struct {
@@ -135,7 +135,7 @@ func (s *Store) StoresSet(opts *pb.StoresSetOptions) error {
 			} else {
 				sample = k.Floats
 			}
-			xlog.Debug("Key is not normalized", "sample", sample)
+			log.Debug().Msgf("Key is not normalized: %v", sample)
 		}
 
 		kvs[i] = Pair{
@@ -238,7 +238,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
 			assert(!hasKey(s.keys, k), fmt.Sprintf("Key exists, but was not found: t=%d, %v", len(tail_ks), k))
 		}
 
-		xlog.Debug("Delete", "found", found, "tailLen", len(tail_ks), "j", j, "mergeKeysLen", len(merge_ks), "mergeValuesLen", len(merge_vs))
+		log.Debug().Msgf("Delete: found = %v, t = %d, j = %d, len(merge_ks) = %d, len(merge_vs) = %d", found, len(tail_ks), j, len(merge_ks), len(merge_vs))
 	}
 
 	merge_ks = append(merge_ks, tail_ks...)
@@ -261,7 +261,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
 	}(), "Keys to delete still present")
 
 	if len(s.keys) != l {
-		xlog.Debug("Delete: Some keys not found", "keysLen", len(s.keys), "expectedLen", l)
+		log.Debug().Msgf("Delete: Some keys not found: len(s.keys) = %d, l = %d", len(s.keys), l)
 	}
 
 	return nil
@@ -273,7 +273,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
 	ks := sortIntoKeySlicese(opts.Keys)
 
 	if len(s.keys) == 0 {
-		xlog.Debug("Get: No keys in store")
+		log.Debug().Msgf("Get: No keys in store")
 	}
 
 	if s.keyLen == -1 {
@@ -305,7 +305,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
 	}
 
 	if len(pbKeys) != len(opts.Keys) {
-		xlog.Debug("Get: Some keys not found", "pbKeysLen", len(pbKeys), "optsKeysLen", len(opts.Keys), "storeKeysLen", len(s.keys))
+		log.Debug().Msgf("Get: Some keys not found: len(pbKeys) = %d, len(opts.Keys) = %d, len(s.Keys) = %d", len(pbKeys), len(opts.Keys), len(s.keys))
 	}
 
 	return pb.StoresGetResult{
@@ -507,7 +507,7 @@ func (s *Store) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, err
 			} else {
 				sample = tk
 			}
-			xlog.Debug("Trying to compare non-normalized key with normalized keys", "sample", sample)
+			log.Debug().Msgf("Trying to compare non-normalized key with normalized keys: %v", sample)
 		}
 
 		return s.StoresFindFallback(opts)

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=4ff2c8c74bd17c2cfffe3a01be77743fb3efba2f
+STABLEDIFFUSION_GGML_VERSION?=0ebe6fe118f125665939b27c89f34ed38716bff8
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/stablediffusion-ggml/gosd.cpp

@@ -1,5 +1,3 @@
-#include "stable-diffusion.h"
-#include <cmath>
 #include <cstdint>
 #define GGML_MAX_NAME 128
 
@@ -8,9 +6,7 @@
 #include <time.h>
 #include <string>
 #include <vector>
-#include <map>
 #include <filesystem>
-#include <algorithm>
 #include "gosd.h"
 
 #define STB_IMAGE_IMPLEMENTATION
@@ -24,13 +20,11 @@
 #define STB_IMAGE_RESIZE_IMPLEMENTATION
 #define STB_IMAGE_RESIZE_STATIC
 #include "stb_image_resize.h"
-#include <stdlib.h>
-#include <regex>
 
 // Names of the sampler method, same order as enum sample_method in stable-diffusion.h
 const char* sample_method_str[] = {
+    "default",
     "euler",
-    "euler_a",
     "heun",
     "dpm2",
     "dpm++2s_a",
@@ -41,384 +35,29 @@ const char* sample_method_str[] = {
     "lcm",
     "ddim_trailing",
     "tcd",
+    "euler_a",
 };
 
 static_assert(std::size(sample_method_str) == SAMPLE_METHOD_COUNT, "sample method mismatch");
 
 // Names of the sigma schedule overrides, same order as sample_schedule in stable-diffusion.h
 const char* schedulers[] = {
+    "default",
     "discrete",
     "karras",
     "exponential",
     "ays",
     "gits",
-    "sgm_uniform",
-    "simple",
     "smoothstep",
-    "kl_optimal",
-    "lcm",
 };
 
-static_assert(std::size(schedulers) == SCHEDULER_COUNT, "schedulers mismatch");
+static_assert(std::size(schedulers) == SCHEDULE_COUNT, "schedulers mismatch");
 
-// New enum string arrays
-const char* rng_type_str[] = {
-    "std_default",
-    "cuda",
-    "cpu",
-};
-static_assert(std::size(rng_type_str) == RNG_TYPE_COUNT, "rng type mismatch");
-
-const char* prediction_str[] = {
-    "epsilon",
-    "v",
-    "edm_v",
-    "flow",
-    "flux_flow",
-    "flux2_flow",
-};
-static_assert(std::size(prediction_str) == PREDICTION_COUNT, "prediction mismatch");
-
-const char* lora_apply_mode_str[] = {
-    "auto",
-    "immediately",
-    "at_runtime",
-};
-static_assert(std::size(lora_apply_mode_str) == LORA_APPLY_MODE_COUNT, "lora apply mode mismatch");
-
-constexpr const char* sd_type_str[] = {
-    "f32",      // 0
-    "f16",      // 1
-    "q4_0",     // 2
-    "q4_1",     // 3
-    nullptr,    // 4
-    nullptr,    // 5
-    "q5_0",     // 6
-    "q5_1",     // 7
-    "q8_0",     // 8
-    "q8_1",     // 9
-    "q2_k",     // 10
-    "q3_k",     // 11
-    "q4_k",     // 12
-    "q5_k",     // 13
-    "q6_k",     // 14
-    "q8_k",     // 15
-    "iq2_xxs",  // 16
-    "iq2_xs",   // 17
-    "iq3_xxs",  // 18
-    "iq1_s",    // 19
-    "iq4_nl",   // 20
-    "iq3_s",    // 21
-    "iq2_s",    // 22
-    "iq4_xs",   // 23
-    "i8",       // 24
-    "i16",      // 25
-    "i32",      // 26
-    "i64",      // 27
-    "f64",      // 28
-    "iq1_m",    // 29
-    "bf16",     // 30
-    nullptr, nullptr, nullptr, nullptr,  // 31-34
-    "tq1_0",    // 35
-    "tq2_0",    // 36
-    nullptr, nullptr,           // 37-38
-    "mxfp4"     // 39
-};
-static_assert(std::size(sd_type_str) == SD_TYPE_COUNT, "sd type mismatch");
-
-sd_ctx_params_t ctx_params;
 sd_ctx_t* sd_c;
 // Moved from the context (load time) to generation time params
-scheduler_t scheduler = SCHEDULER_COUNT;
-sample_method_t sample_method = SAMPLE_METHOD_COUNT;
+scheduler_t scheduler = scheduler_t::DEFAULT;
 
-// Storage for embeddings (needs to persist for the lifetime of ctx_params)
-static std::vector<sd_embedding_t> embedding_vec;
-// Storage for embedding strings (needs to persist as long as embedding_vec references them)
-static std::vector<std::string> embedding_strings;
-
-// Storage for LoRAs (needs to persist for the lifetime of generation params)
-static std::vector<sd_lora_t> lora_vec;
-// Storage for LoRA strings (needs to persist as long as lora_vec references them)
-static std::vector<std::string> lora_strings;
-// Storage for lora_dir path
-static std::string lora_dir_path;
-
-// Build embeddings vector from directory, similar to upstream CLI
-static void build_embedding_vec(const char* embedding_dir) {
-    embedding_vec.clear();
-    embedding_strings.clear();
-
-    if (!embedding_dir || strlen(embedding_dir) == 0) {
-        return;
-    }
-
-    if (!std::filesystem::exists(embedding_dir) || !std::filesystem::is_directory(embedding_dir)) {
-        fprintf(stderr, "Embedding directory does not exist or is not a directory: %s\n", embedding_dir);
-        return;
-    }
-
-    static const std::vector<std::string> valid_ext = {".pt", ".safetensors", ".gguf"};
-
-    for (const auto& entry : std::filesystem::directory_iterator(embedding_dir)) {
-        if (!entry.is_regular_file()) {
-            continue;
-        }
-
-        auto path = entry.path();
-        std::string ext = path.extension().string();
-
-        bool valid = false;
-        for (const auto& e : valid_ext) {
-            if (ext == e) {
-                valid = true;
-                break;
-            }
-        }
-        if (!valid) {
-            continue;
-        }
-
-        std::string name = path.stem().string();
-        std::string full_path = path.string();
-
-        // Store strings in persistent storage
-        embedding_strings.push_back(name);
-        embedding_strings.push_back(full_path);
-
-        sd_embedding_t item;
-        item.name = embedding_strings[embedding_strings.size() - 2].c_str();
-        item.path = embedding_strings[embedding_strings.size() - 1].c_str();
-
-        embedding_vec.push_back(item);
-        fprintf(stderr, "Found embedding: %s -> %s\n", item.name, item.path);
-    }
-
-    fprintf(stderr, "Loaded %zu embeddings from %s\n", embedding_vec.size(), embedding_dir);
-}
-
-// Discover LoRA files in directory and build a map of name -> path
-static std::map<std::string, std::string> discover_lora_files(const char* lora_dir) {
-    std::map<std::string, std::string> lora_map;
-
-    if (!lora_dir || strlen(lora_dir) == 0) {
-        fprintf(stderr, "LoRA directory not specified\n");
-        return lora_map;
-    }
-
-    if (!std::filesystem::exists(lora_dir) || !std::filesystem::is_directory(lora_dir)) {
-        fprintf(stderr, "LoRA directory does not exist or is not a directory: %s\n", lora_dir);
-        return lora_map;
-    }
-
-    static const std::vector<std::string> valid_ext = {".safetensors", ".ckpt", ".pt", ".gguf"};
-
-    fprintf(stderr, "Discovering LoRA files in: %s\n", lora_dir);
-
-    for (const auto& entry : std::filesystem::directory_iterator(lora_dir)) {
-        if (!entry.is_regular_file()) {
-            continue;
-        }
-
-        auto path = entry.path();
-        std::string ext = path.extension().string();
-
-        bool valid = false;
-        for (const auto& e : valid_ext) {
-            if (ext == e) {
-                valid = true;
-                break;
-            }
-        }
-        if (!valid) {
-            continue;
-        }
-
-        std::string name = path.stem().string();  // stem() already removes extension
-        std::string full_path = path.string();
-
-        // Store the name (without extension) -> full path mapping
-        // This allows users to specify just the name in <lora:name:strength>
-        lora_map[name] = full_path;
-
-        fprintf(stderr, "Found LoRA file: %s -> %s\n", name.c_str(), full_path.c_str());
-    }
-
-    fprintf(stderr, "Discovered %zu LoRA files in %s\n", lora_map.size(), lora_dir);
-    return lora_map;
-}
-
-// Helper function to check if a path is absolute (matches upstream)
-static bool is_absolute_path(const std::string& p) {
-#ifdef _WIN32
-    // Windows: C:/path or C:\path
-    return p.size() > 1 && std::isalpha(static_cast<unsigned char>(p[0])) && p[1] == ':';
-#else
-    // Unix: /path
-    return !p.empty() && p[0] == '/';
-#endif
-}
-
-// Parse LoRAs from prompt string (e.g., "<lora:name:1.0>" or "<lora:name>")
-// Returns a vector of LoRA info and the cleaned prompt with LoRA tags removed
-// Matches upstream implementation more closely
-static std::pair<std::vector<sd_lora_t>, std::string> parse_loras_from_prompt(const std::string& prompt, const char* lora_dir) {
-    std::vector<sd_lora_t> loras;
-    std::string cleaned_prompt = prompt;
-
-    if (!lora_dir || strlen(lora_dir) == 0) {
-        fprintf(stderr, "LoRA directory not set, cannot parse LoRAs from prompt\n");
-        return {loras, cleaned_prompt};
-    }
-
-    // Discover LoRA files for name-based lookup
-    std::map<std::string, std::string> discovered_lora_map = discover_lora_files(lora_dir);
-
-    // Map to accumulate multipliers for the same LoRA (matches upstream)
-    std::map<std::string, float> lora_map;
-    std::map<std::string, float> high_noise_lora_map;
-
-    static const std::regex re(R"(<lora:([^:>]+):([^>]+)>)");
-    static const std::vector<std::string> valid_ext = {".pt", ".safetensors", ".gguf"};
-    std::smatch m;
-
-    std::string tmp = prompt;
-
-    fprintf(stderr, "Parsing LoRAs from prompt: %s\n", prompt.c_str());
-
-    while (std::regex_search(tmp, m, re)) {
-        std::string raw_path = m[1].str();
-        const std::string raw_mul = m[2].str();
-
-        float mul = 0.f;
-        try {
-            mul = std::stof(raw_mul);
-        } catch (...) {
-            tmp = m.suffix().str();
-            cleaned_prompt = std::regex_replace(cleaned_prompt, re, "", std::regex_constants::format_first_only);
-            fprintf(stderr, "Invalid LoRA multiplier '%s', skipping\n", raw_mul.c_str());
-            continue;
-        }
-
-        bool is_high_noise = false;
-        static const std::string prefix = "|high_noise|";
-        if (raw_path.rfind(prefix, 0) == 0) {
-            raw_path.erase(0, prefix.size());
-            is_high_noise = true;
-        }
-
-        std::filesystem::path final_path;
-        if (is_absolute_path(raw_path)) {
-            final_path = raw_path;
-        } else {
-            // Try name-based lookup first
-            auto it = discovered_lora_map.find(raw_path);
-            if (it != discovered_lora_map.end()) {
-                final_path = it->second;
-            } else {
-                // Try case-insensitive lookup
-                bool found = false;
-                for (const auto& pair : discovered_lora_map) {
-                    std::string lower_name = raw_path;
-                    std::string lower_key = pair.first;
-                    std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(), ::tolower);
-                    std::transform(lower_key.begin(), lower_key.end(), lower_key.begin(), ::tolower);
-                    if (lower_name == lower_key) {
-                        final_path = pair.second;
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found) {
-                    // Try as relative path in lora_dir
-                    final_path = std::filesystem::path(lora_dir) / raw_path;
-                }
-            }
-        }
-
-        // Try adding extensions if file doesn't exist
-        if (!std::filesystem::exists(final_path)) {
-            bool found = false;
-            for (const auto& ext : valid_ext) {
-                std::filesystem::path try_path = final_path;
-                try_path += ext;
-                if (std::filesystem::exists(try_path)) {
-                    final_path = try_path;
-                    found = true;
-                    break;
-                }
-            }
-            if (!found) {
-                fprintf(stderr, "WARNING: LoRA file not found: %s\n", final_path.lexically_normal().string().c_str());
-                tmp = m.suffix().str();
-                cleaned_prompt = std::regex_replace(cleaned_prompt, re, "", std::regex_constants::format_first_only);
-                continue;
-            }
-        }
-
-        // Normalize path (matches upstream)
-        const std::string key = final_path.lexically_normal().string();
-
-        // Accumulate multiplier if same LoRA appears multiple times (matches upstream)
-        if (is_high_noise) {
-            high_noise_lora_map[key] += mul;
-        } else {
-            lora_map[key] += mul;
-        }
-
-        fprintf(stderr, "Parsed LoRA: path='%s', multiplier=%.2f, is_high_noise=%s\n",
-                key.c_str(), mul, is_high_noise ? "true" : "false");
-
-        cleaned_prompt = std::regex_replace(cleaned_prompt, re, "", std::regex_constants::format_first_only);
-        tmp = m.suffix().str();
-    }
-
-    // Build final LoRA vector from accumulated maps (matches upstream)
-    // Store all path strings first to ensure they persist
-    for (const auto& kv : lora_map) {
-        lora_strings.push_back(kv.first);
-    }
-    for (const auto& kv : high_noise_lora_map) {
-        lora_strings.push_back(kv.first);
-    }
-
-    // Now build the LoRA vector with pointers to the stored strings
-    size_t string_idx = 0;
-    for (const auto& kv : lora_map) {
-        sd_lora_t item;
-        item.is_high_noise = false;
-        item.path = lora_strings[string_idx].c_str();
-        item.multiplier = kv.second;
-        loras.push_back(item);
-        string_idx++;
-    }
-
-    for (const auto& kv : high_noise_lora_map) {
-        sd_lora_t item;
-        item.is_high_noise = true;
-        item.path = lora_strings[string_idx].c_str();
-        item.multiplier = kv.second;
-        loras.push_back(item);
-        string_idx++;
-    }
-
-    // Clean up extra spaces
-    std::regex space_regex(R"(\s+)");
-    cleaned_prompt = std::regex_replace(cleaned_prompt, space_regex, " ");
-    // Trim leading/trailing spaces
-    size_t first = cleaned_prompt.find_first_not_of(" \t");
-    if (first != std::string::npos) {
-        cleaned_prompt.erase(0, first);
-    }
-    size_t last = cleaned_prompt.find_last_not_of(" \t");
-    if (last != std::string::npos) {
-        cleaned_prompt.erase(last + 1);
-    }
-
-    fprintf(stderr, "Parsed %zu LoRA(s) from prompt. Cleaned prompt: %s\n", loras.size(), cleaned_prompt.c_str());
-
-    return {loras, cleaned_prompt};
-}
+sample_method_t sample_method;
 
 // Copied from the upstream CLI
 static void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
@@ -459,7 +98,7 @@ int load_model(const char *model, char *model_path, char* options[], int threads
 
     const char *stableDiffusionModel = "";
     if (diff == 1 ) {
-        stableDiffusionModel = strdup(model);
+        stableDiffusionModel = model;
         model = "";
     }
 
@@ -470,38 +109,8 @@ int load_model(const char *model, char *model_path, char* options[], int threads
     const char *vae_path  = "";
     const char *scheduler_str = "";
     const char *sampler = "";
-    const char *clip_vision_path = "";
-    const char *llm_path = "";
-    const char *llm_vision_path = "";
-    const char *diffusion_model_path = stableDiffusionModel;
-    const char *high_noise_diffusion_model_path = "";
-    const char *taesd_path  = "";
-    const char *control_net_path = "";
-    const char *embedding_dir = "";
-    const char *photo_maker_path = "";
-    const char *tensor_type_rules = "";
     char *lora_dir = model_path;
-
-    bool vae_decode_only = true;
-    int n_threads = threads;
-    enum sd_type_t wtype = SD_TYPE_COUNT;
-    enum rng_type_t rng_type = CUDA_RNG;
-    enum rng_type_t sampler_rng_type = RNG_TYPE_COUNT;
-    enum prediction_t prediction = PREDICTION_COUNT;
-    enum lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO;
-    bool offload_params_to_cpu = false;
-    bool keep_clip_on_cpu = false;
-    bool keep_control_net_on_cpu = false;
-    bool keep_vae_on_cpu = false;
-    bool diffusion_flash_attn = false;
-    bool tae_preview_only = false;
-    bool diffusion_conv_direct = false;
-    bool vae_conv_direct = false;
-    bool force_sdxl_vae_conv_scale = false;
-    bool chroma_use_dit_mask = true;
-    bool chroma_use_t5_mask = false;
-    int chroma_t5_mask_pad = 1;
-    float flow_shift = INFINITY;
+    bool lora_dir_allocated = false;
 
     fprintf(stderr, "parsing options: %p\n", options);
 
@@ -514,16 +123,16 @@ int load_model(const char *model, char *model_path, char* options[], int threads
         }
 
         if (!strcmp(optname, "clip_l_path")) {
-            clip_l_path = strdup(optval);
+            clip_l_path = optval;
         }
         if (!strcmp(optname, "clip_g_path")) {
-            clip_g_path = strdup(optval);
+            clip_g_path = optval;
         }
         if (!strcmp(optname, "t5xxl_path")) {
-            t5xxl_path = strdup(optval);
+            t5xxl_path = optval;
         }
         if (!strcmp(optname, "vae_path")) {
-            vae_path = strdup(optval);
+            vae_path = optval;
         }
         if (!strcmp(optname, "scheduler")) {
             scheduler_str = optval;
@@ -538,201 +147,18 @@ int load_model(const char *model, char *model_path, char* options[], int threads
                 std::filesystem::path lora_path(optval);
                 std::filesystem::path full_lora_path = model_path_str / lora_path;
                 lora_dir = strdup(full_lora_path.string().c_str());
-                lora_dir_path = full_lora_path.string();
-                fprintf(stderr, "LoRA dir resolved to: %s\n", lora_dir);
+                lora_dir_allocated = true;
+                fprintf(stderr, "Lora dir resolved to: %s\n", lora_dir);
             } else {
                 lora_dir = strdup(optval);
-                lora_dir_path = std::string(optval);
+                lora_dir_allocated = true;
                 fprintf(stderr, "No model path provided, using lora dir as-is: %s\n", lora_dir);
             }
-            // Discover LoRAs immediately when directory is set
-            if (lora_dir && strlen(lora_dir) > 0) {
-                discover_lora_files(lora_dir);
-            }
-        }
-
-        // New parsing
-        if (!strcmp(optname, "clip_vision_path")) clip_vision_path = strdup(optval);
-        if (!strcmp(optname, "llm_path")) llm_path = strdup(optval);
-        if (!strcmp(optname, "llm_vision_path")) llm_vision_path = strdup(optval);
-        if (!strcmp(optname, "diffusion_model_path")) diffusion_model_path = strdup(optval);
-        if (!strcmp(optname, "high_noise_diffusion_model_path")) high_noise_diffusion_model_path = strdup(optval);
-        if (!strcmp(optname, "taesd_path")) taesd_path = strdup(optval);
-        if (!strcmp(optname, "control_net_path")) control_net_path = strdup(optval);
-        if (!strcmp(optname, "embedding_dir")) {
-            // Path join with model dir
-            if (model_path && strlen(model_path) > 0) {
-                std::filesystem::path model_path_str(model_path);
-                std::filesystem::path embedding_path(optval);
-                std::filesystem::path full_embedding_path = model_path_str / embedding_path;
-                embedding_dir = strdup(full_embedding_path.string().c_str());
-                fprintf(stderr, "Embedding dir resolved to: %s\n", embedding_dir);
-            } else {
-                embedding_dir = strdup(optval);
-                fprintf(stderr, "No model path provided, using embedding dir as-is: %s\n", embedding_dir);
-            }
-        }
-        if (!strcmp(optname, "photo_maker_path")) photo_maker_path = strdup(optval);
-        if (!strcmp(optname, "tensor_type_rules")) tensor_type_rules = strdup(optval);
-
-        if (!strcmp(optname, "vae_decode_only")) vae_decode_only = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "offload_params_to_cpu")) offload_params_to_cpu = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "keep_clip_on_cpu")) keep_clip_on_cpu = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "keep_control_net_on_cpu")) keep_control_net_on_cpu = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "keep_vae_on_cpu")) keep_vae_on_cpu = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "diffusion_flash_attn")) diffusion_flash_attn = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "tae_preview_only")) tae_preview_only = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "diffusion_conv_direct")) diffusion_conv_direct = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "vae_conv_direct")) vae_conv_direct = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "force_sdxl_vae_conv_scale")) force_sdxl_vae_conv_scale = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "chroma_use_dit_mask")) chroma_use_dit_mask = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-        if (!strcmp(optname, "chroma_use_t5_mask")) chroma_use_t5_mask = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
-
-        if (!strcmp(optname, "n_threads")) n_threads = atoi(optval);
-        if (!strcmp(optname, "chroma_t5_mask_pad")) chroma_t5_mask_pad = atoi(optval);
-
-        if (!strcmp(optname, "flow_shift")) flow_shift = atof(optval);
-
-        if (!strcmp(optname, "rng_type")) {
-            int found = -1;
-            for (int m = 0; m < RNG_TYPE_COUNT; m++) {
-                if (!strcmp(optval, rng_type_str[m])) {
-                    found = m;
-                    break;
-                }
-            }
-            if (found != -1) {
-                rng_type = (rng_type_t)found;
-                fprintf(stderr, "Found rng_type: %s\n", optval);
-            } else {
-                fprintf(stderr, "Invalid rng_type: %s, using default\n", optval);
-            }
-        }
-        if (!strcmp(optname, "sampler_rng_type")) {
-            int found = -1;
-            for (int m = 0; m < RNG_TYPE_COUNT; m++) {
-                if (!strcmp(optval, rng_type_str[m])) {
-                    found = m;
-                    break;
-                }
-            }
-            if (found != -1) {
-                sampler_rng_type = (rng_type_t)found;
-                fprintf(stderr, "Found sampler_rng_type: %s\n", optval);
-            } else {
-                fprintf(stderr, "Invalid sampler_rng_type: %s, using default\n", optval);
-            }
-        }
-        if (!strcmp(optname, "prediction")) {
-            int found = -1;
-            for (int m = 0; m < PREDICTION_COUNT; m++) {
-                if (!strcmp(optval, prediction_str[m])) {
-                    found = m;
-                    break;
-                }
-            }
-            if (found != -1) {
-                prediction = (prediction_t)found;
-                fprintf(stderr, "Found prediction: %s\n", optval);
-            } else {
-                fprintf(stderr, "Invalid prediction: %s, using default\n", optval);
-            }
-        }
-        if (!strcmp(optname, "lora_apply_mode")) {
-            int found = -1;
-            for (int m = 0; m < LORA_APPLY_MODE_COUNT; m++) {
-                if (!strcmp(optval, lora_apply_mode_str[m])) {
-                    found = m;
-                    break;
-                }
-            }
-            if (found != -1) {
-                lora_apply_mode = (lora_apply_mode_t)found;
-                fprintf(stderr, "Found lora_apply_mode: %s\n", optval);
-            } else {
-                fprintf(stderr, "Invalid lora_apply_mode: %s, using default\n", optval);
-            }
-        }
-        if (!strcmp(optname, "wtype")) {
-            int found = -1;
-            for (int m = 0; m < SD_TYPE_COUNT; m++) {
-                if (sd_type_str[m] && !strcmp(optval, sd_type_str[m])) {
-                    found = m;
-                    break;
-                }
-            }
-            if (found != -1) {
-                wtype = (sd_type_t)found;
-                fprintf(stderr, "Found wtype: %s\n", optval);
-            } else {
-                fprintf(stderr, "Invalid wtype: %s, using default\n", optval);
-            }
         }
     }
 
     fprintf(stderr, "parsed options\n");
 
-    // Build embeddings vector from directory if provided
-    build_embedding_vec(embedding_dir);
-
-    fprintf (stderr, "Creating context\n");
-    sd_ctx_params_init(&ctx_params);
-    ctx_params.model_path = model;
-    ctx_params.clip_l_path = clip_l_path;
-    ctx_params.clip_g_path = clip_g_path;
-    ctx_params.clip_vision_path = clip_vision_path;
-    ctx_params.t5xxl_path = t5xxl_path;
-    ctx_params.llm_path = llm_path;
-    ctx_params.llm_vision_path = llm_vision_path;
-    ctx_params.diffusion_model_path = diffusion_model_path;
-    ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path;
-    ctx_params.vae_path = vae_path;
-    ctx_params.taesd_path = taesd_path;
-    ctx_params.control_net_path = control_net_path;
-    if (lora_dir && strlen(lora_dir) > 0) {
-        lora_dir_path = std::string(lora_dir);
-        fprintf(stderr, "LoRA model directory set to: %s\n", lora_dir);
-        // Discover LoRAs at load time for logging
-        discover_lora_files(lora_dir);
-    } else {
-        fprintf(stderr, "WARNING: LoRA model directory not set. LoRAs in prompts will not be loaded.\n");
-    }
-    // Set embeddings array and count
-    ctx_params.embeddings = embedding_vec.empty() ? NULL : embedding_vec.data();
-    ctx_params.embedding_count = static_cast<uint32_t>(embedding_vec.size());
-    ctx_params.photo_maker_path = photo_maker_path;
-    ctx_params.tensor_type_rules = tensor_type_rules;
-    ctx_params.vae_decode_only = vae_decode_only;
-    // XXX: Setting to true causes a segfault on the second run
-    ctx_params.free_params_immediately = false;
-    ctx_params.n_threads = n_threads;
-    ctx_params.rng_type = rng_type;
-    ctx_params.keep_clip_on_cpu = keep_clip_on_cpu;
-    if (wtype != SD_TYPE_COUNT) ctx_params.wtype = wtype;
-    if (sampler_rng_type != RNG_TYPE_COUNT) ctx_params.sampler_rng_type = sampler_rng_type;
-    if (prediction != PREDICTION_COUNT) ctx_params.prediction = prediction;
-    if (lora_apply_mode != LORA_APPLY_MODE_COUNT) ctx_params.lora_apply_mode = lora_apply_mode;
-    ctx_params.offload_params_to_cpu = offload_params_to_cpu;
-    ctx_params.keep_control_net_on_cpu = keep_control_net_on_cpu;
-    ctx_params.keep_vae_on_cpu = keep_vae_on_cpu;
-    ctx_params.diffusion_flash_attn = diffusion_flash_attn;
-    ctx_params.tae_preview_only = tae_preview_only;
-    ctx_params.diffusion_conv_direct = diffusion_conv_direct;
-    ctx_params.vae_conv_direct = vae_conv_direct;
-    ctx_params.force_sdxl_vae_conv_scale = force_sdxl_vae_conv_scale;
-    ctx_params.chroma_use_dit_mask = chroma_use_dit_mask;
-    ctx_params.chroma_use_t5_mask = chroma_use_t5_mask;
-    ctx_params.chroma_t5_mask_pad = chroma_t5_mask_pad;
-    ctx_params.flow_shift = flow_shift;
-    sd_ctx_t* sd_ctx = new_sd_ctx(&ctx_params);
-
-    if (sd_ctx == NULL) {
-        fprintf (stderr, "failed loading model (generic error)\n");
-        // TODO: Clean up allocated memory
-        return 1;
-    }
-    fprintf (stderr, "Created context: OK\n");
-
     int sample_method_found = -1;
     for (int m = 0; m < SAMPLE_METHOD_COUNT; m++) {
         if (!strcmp(sampler, sample_method_str[m])) {
@@ -741,24 +167,54 @@ int load_model(const char *model, char *model_path, char* options[], int threads
         }
     }
     if (sample_method_found == -1) {
-        sample_method_found = sd_get_default_sample_method(sd_ctx);
-        fprintf(stderr, "Invalid sample method, using default: %s\n", sample_method_str[sample_method_found]);
+        fprintf(stderr, "Invalid sample method, default to EULER_A!\n");
+        sample_method_found = sample_method_t::SAMPLE_METHOD_DEFAULT;
     }
     sample_method = (sample_method_t)sample_method_found;
 
-    for (int d = 0; d < SCHEDULER_COUNT; d++) {
+    for (int d = 0; d < SCHEDULE_COUNT; d++) {
         if (!strcmp(scheduler_str, schedulers[d])) {
             scheduler = (scheduler_t)d;
             fprintf (stderr, "Found scheduler: %s\n", scheduler_str);
         }
     }
-    if (scheduler == SCHEDULER_COUNT) {
-      scheduler = sd_get_default_scheduler(sd_ctx, sample_method);
-      fprintf(stderr, "Invalid scheduler, using default: %s\n", schedulers[scheduler]);
+
+    fprintf (stderr, "Creating context\n");
+    sd_ctx_params_t ctx_params;
+    sd_ctx_params_init(&ctx_params);
+    ctx_params.model_path = model;
+    ctx_params.clip_l_path = clip_l_path;
+    ctx_params.clip_g_path = clip_g_path;
+    ctx_params.t5xxl_path = t5xxl_path;
+    ctx_params.diffusion_model_path = stableDiffusionModel;
+    ctx_params.vae_path = vae_path;
+    ctx_params.taesd_path = "";
+    ctx_params.control_net_path = "";
+    ctx_params.lora_model_dir = lora_dir;
+    ctx_params.embedding_dir = "";
+    ctx_params.vae_decode_only = false;
+    ctx_params.free_params_immediately = false;
+    ctx_params.n_threads = threads;
+    ctx_params.rng_type = STD_DEFAULT_RNG;
+    sd_ctx_t* sd_ctx = new_sd_ctx(&ctx_params);
+
+    if (sd_ctx == NULL) {
+        fprintf (stderr, "failed loading model (generic error)\n");
+        // Clean up allocated memory
+        if (lora_dir_allocated && lora_dir) {
+            free(lora_dir);
+        }
+        return 1;
     }
+    fprintf (stderr, "Created context: OK\n");
 
     sd_c = sd_ctx;
 
+    // Clean up allocated memory
+    if (lora_dir_allocated && lora_dir) {
+        free(lora_dir);
+    }
+
     return 0;
 }
 
@@ -787,66 +243,12 @@ sd_tiling_params_t* sd_img_gen_params_get_vae_tiling_params(sd_img_gen_params_t
 sd_img_gen_params_t* sd_img_gen_params_new(void) {
     sd_img_gen_params_t *params = (sd_img_gen_params_t *)std::malloc(sizeof(sd_img_gen_params_t));
     sd_img_gen_params_init(params);
-    sd_sample_params_init(&params->sample_params);
-    sd_cache_params_init(&params->cache);
-    params->control_strength = 0.9f;
     return params;
 }
 
-// Storage for cleaned prompt strings (needs to persist)
-static std::string cleaned_prompt_storage;
-static std::string cleaned_negative_prompt_storage;
-
 void sd_img_gen_params_set_prompts(sd_img_gen_params_t *params, const char *prompt, const char *negative_prompt) {
-    // Clear previous LoRA data
-    lora_vec.clear();
-    lora_strings.clear();
-
-    // Parse LoRAs from prompt
-    std::string prompt_str = prompt ? prompt : "";
-    std::string negative_prompt_str = negative_prompt ? negative_prompt : "";
-
-    // Get lora_dir from ctx_params if available, otherwise use stored path
-    const char* lora_dir_to_use = lora_dir_path.empty() ? nullptr : lora_dir_path.c_str();
-
-    auto [loras, cleaned_prompt] = parse_loras_from_prompt(prompt_str, lora_dir_to_use);
-    lora_vec = loras;
-    cleaned_prompt_storage = cleaned_prompt;
-
-    // Also check negative prompt for LoRAs (though this is less common)
-    auto [neg_loras, cleaned_negative] = parse_loras_from_prompt(negative_prompt_str, lora_dir_to_use);
-    // Merge negative prompt LoRAs (though typically not used)
-    if (!neg_loras.empty()) {
-        fprintf(stderr, "Note: Found %zu LoRAs in negative prompt (may not be supported)\n", neg_loras.size());
-    }
-    cleaned_negative_prompt_storage = cleaned_negative;
-
-    // Set the cleaned prompts
-    params->prompt = cleaned_prompt_storage.c_str();
-    params->negative_prompt = cleaned_negative_prompt_storage.c_str();
-
-    // Set LoRAs in params
-    params->loras = lora_vec.empty() ? nullptr : lora_vec.data();
-    params->lora_count = static_cast<uint32_t>(lora_vec.size());
-
-    fprintf(stderr, "Set prompts with %zu LoRAs. Original prompt: %s\n", lora_vec.size(), prompt ? prompt : "(null)");
-    fprintf(stderr, "Cleaned prompt: %s\n", cleaned_prompt_storage.c_str());
-
-    // Debug: Verify LoRAs are set correctly
-    if (params->loras && params->lora_count > 0) {
-        fprintf(stderr, "DEBUG: LoRAs set in params structure:\n");
-        for (uint32_t i = 0; i < params->lora_count; i++) {
-            fprintf(stderr, "  params->loras[%u]: path='%s' (ptr=%p), multiplier=%.2f, is_high_noise=%s\n",
-                    i,
-                    params->loras[i].path ? params->loras[i].path : "(null)",
-                    (void*)params->loras[i].path,
-                    params->loras[i].multiplier,
-                    params->loras[i].is_high_noise ? "true" : "false");
-        }
-    } else {
-        fprintf(stderr, "DEBUG: No LoRAs set in params structure (loras=%p, lora_count=%u)\n",
-                (void*)params->loras, params->lora_count);
-    }
+    params->prompt = prompt;
+    params->negative_prompt = negative_prompt;
 }
 
 void sd_img_gen_params_set_dimensions(sd_img_gen_params_t *params, int width, int height) {
@@ -858,7 +260,7 @@ void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed) {
     params->seed = seed;
 }
 
-int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count) {
+int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char **ref_images, int ref_images_count) {
 
     sd_image_t* results;
 
@@ -1038,24 +440,6 @@ int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, cha
         }
     }
 
-    // Log LoRA information
-    if (p->loras && p->lora_count > 0) {
-        fprintf(stderr, "Using %u LoRA(s) in generation:\n", p->lora_count);
-        for (uint32_t i = 0; i < p->lora_count; i++) {
-            fprintf(stderr, "  LoRA[%u]: path='%s', multiplier=%.2f, is_high_noise=%s\n",
-                    i,
-                    p->loras[i].path ? p->loras[i].path : "(null)",
-                    p->loras[i].multiplier,
-                    p->loras[i].is_high_noise ? "true" : "false");
-        }
-    } else {
-        fprintf(stderr, "No LoRAs specified for this generation\n");
-    }
-
-    fprintf(stderr, "Generating image with params: \nctx\n---\n%s\ngen\n---\n%s\n",
-            sd_ctx_params_to_str(&ctx_params),
-            sd_img_gen_params_to_str(p));
-
     results = generate_image(sd_c, p);
 
     std::free(p);
@@ -1088,12 +472,9 @@ int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, cha
     fprintf (stderr, "Channel: %d\n", results[0].channel);
     fprintf (stderr, "Data: %p\n", results[0].data);
 
-    int ret = stbi_write_png(dst, results[0].width, results[0].height, results[0].channel,
-                             results[0].data, 0, NULL);
-    if (ret)
-      fprintf (stderr, "Saved resulting image to '%s'\n", dst);
-    else
-      fprintf(stderr, "Failed to write image to '%s'\n", dst);
+    stbi_write_png(dst, results[0].width, results[0].height, results[0].channel,
+                       results[0].data, 0, NULL);
+    fprintf (stderr, "Saved resulting image to '%s'\n", dst);
 
     // Clean up
     free(results[0].data);
@@ -1104,14 +485,12 @@ int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, cha
     for (auto buffer : ref_image_buffers) {
         if (buffer) free(buffer);
     }
-    fprintf (stderr, "gen_image is done: %s\n", dst);
-    fflush(stderr);
+    fprintf (stderr, "gen_image is done: %s", dst);
 
-    return !ret;
+    return 0;
 }
 
 int unload() {
     free_sd_ctx(sd_c);
     return 0;
 }
-

backend/go/stablediffusion-ggml/gosd.go

@@ -22,7 +22,7 @@ type SDGGML struct {
 
 var (
 	LoadModel func(model, model_apth string, options []uintptr, threads int32, diff int) int
-	GenImage  func(params uintptr, steps int, dst string, cfgScale float32, srcImage string, strength float32, maskImage string, refImages []uintptr, refImagesCount int) int
+	GenImage  func(params uintptr, steps int, dst string, cfgScale float32, srcImage string, strength float32, maskImage string, refImages []string, refImagesCount int) int
 
 	TilingParamsSetEnabled       func(params uintptr, enabled bool)
 	TilingParamsSetTileSizes     func(params uintptr, tileSizeX int, tileSizeY int)
@@ -95,12 +95,12 @@ func (sd *SDGGML) Load(opts *pb.ModelOptions) error {
 	sd.cfgScale = opts.CFGScale
 
 	ret := LoadModel(modelFile, modelPathC, options, opts.Threads, diffusionModel)
-	runtime.KeepAlive(keepAlive)
-	fmt.Fprintf(os.Stderr, "LoadModel: %d\n", ret)
 	if ret != 0 {
 		return fmt.Errorf("could not load model")
 	}
 
+	runtime.KeepAlive(keepAlive)
+
 	return nil
 }
 
@@ -123,15 +123,10 @@ func (sd *SDGGML) GenerateImage(opts *pb.GenerateImageRequest) error {
 		}
 	}
 
-	// At the time of writing Purego doesn't recurse into slices and convert Go strings to pointers so we need to do that
-	var keepAlive []any
 	refImagesCount := len(opts.RefImages)
-	refImages := make([]uintptr, refImagesCount, refImagesCount+1)
-	for i, ri := range opts.RefImages {
-		bytep := CString(ri)
-		refImages[i] = uintptr(unsafe.Pointer(bytep))
-		keepAlive = append(keepAlive, bytep)
-	}
+	refImages := make([]string, refImagesCount, refImagesCount+1)
+	copy(refImages, opts.RefImages)
+	*(*uintptr)(unsafe.Add(unsafe.Pointer(&refImages), refImagesCount)) = 0
 
 	// Default strength for img2img (0.75 is a good default)
 	strength := float32(0.75)
@@ -145,8 +140,6 @@ func (sd *SDGGML) GenerateImage(opts *pb.GenerateImageRequest) error {
 	TilingParamsSetEnabled(vaep, false)
 
 	ret := GenImage(p, int(opts.Step), dst, sd.cfgScale, srcImage, strength, maskImage, refImages, refImagesCount)
-	runtime.KeepAlive(keepAlive)
-	fmt.Fprintf(os.Stderr, "GenImage: %d\n", ret)
 	if ret != 0 {
 		return fmt.Errorf("inference failed")
 	}

backend/go/stablediffusion-ggml/gosd.h

@@ -17,7 +17,7 @@ void sd_img_gen_params_set_dimensions(sd_img_gen_params_t *params, int width, in
 void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed);
 
 int load_model(const char *model, char *model_path, char* options[], int threads, int diffusionModel);
-int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count);
+int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char **ref_images, int ref_images_count);
 #ifdef __cplusplus
 }
 #endif

backend/go/whisper/.gitignore

@@ -3,5 +3,5 @@ sources/
 build/
 package/
 whisper
-*.so
-compile_commands.json
+libgowhisper.so
+

backend/go/whisper/Makefile

@@ -8,8 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=e9898ddfb908ffaa7026c66852a023889a5a7202
-SO_TARGET?=libgowhisper.so
+WHISPER_CPP_VERSION?=44fa2f647cf2a6953493b21ab83b50d5f5dbc483
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
 
@@ -58,18 +57,15 @@ sources/whisper.cpp:
 	git checkout $(WHISPER_CPP_VERSION) && \
 	git submodule update --init --recursive --depth 1 --single-branch
 
-# Detect OS
-UNAME_S := $(shell uname -s)
+libgowhisper.so: sources/whisper.cpp CMakeLists.txt gowhisper.cpp gowhisper.h
+	mkdir -p build && \
+	cd build && \
+	cmake .. $(CMAKE_ARGS) && \
+	cmake --build . --config Release -j$(JOBS) && \
+	cd .. && \
+	mv build/libgowhisper.so ./
 
-# Only build CPU variants on Linux
-ifeq ($(UNAME_S),Linux)
-	VARIANT_TARGETS = libgowhisper-avx.so libgowhisper-avx2.so libgowhisper-avx512.so libgowhisper-fallback.so
-else
-	# On non-Linux (e.g., Darwin), build only fallback variant
-	VARIANT_TARGETS = libgowhisper-fallback.so
-endif
-
-whisper: main.go gowhisper.go $(VARIANT_TARGETS)
+whisper: main.go gowhisper.go libgowhisper.so
 	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o whisper ./
 
 package: whisper
@@ -77,46 +73,5 @@ package: whisper
 
 build: package
 
-clean: purge
-	rm -rf libgowhisper*.so sources/whisper.cpp whisper
-
-purge:
-	rm -rf build*
-
-# Build all variants (Linux only)
-ifeq ($(UNAME_S),Linux)
-libgowhisper-avx.so: sources/whisper.cpp
-	$(MAKE) purge
-	$(info ${GREEN}I whisper build info:avx${RESET})
-	SO_TARGET=libgowhisper-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" $(MAKE) libgowhisper-custom
-	rm -rfv build*
-
-libgowhisper-avx2.so: sources/whisper.cpp
-	$(MAKE) purge
-	$(info ${GREEN}I whisper build info:avx2${RESET})
-	SO_TARGET=libgowhisper-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) libgowhisper-custom
-	rm -rfv build*
-
-libgowhisper-avx512.so: sources/whisper.cpp
-	$(MAKE) purge
-	$(info ${GREEN}I whisper build info:avx512${RESET})
-	SO_TARGET=libgowhisper-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) libgowhisper-custom
-	rm -rfv build*
-endif
-
-# Build fallback variant (all platforms)
-libgowhisper-fallback.so: sources/whisper.cpp
-	$(MAKE) purge
-	$(info ${GREEN}I whisper build info:fallback${RESET})
-	SO_TARGET=libgowhisper-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" $(MAKE) libgowhisper-custom
-	rm -rfv build*
-
-libgowhisper-custom: CMakeLists.txt gowhisper.cpp gowhisper.h
-	mkdir -p build-$(SO_TARGET) && \
-	cd build-$(SO_TARGET) && \
-	cmake .. $(CMAKE_ARGS) && \
-	cmake --build . --config Release -j$(JOBS) && \
-	cd .. && \
-	mv build-$(SO_TARGET)/libgowhisper.so ./$(SO_TARGET)
-
-all: whisper package
+clean:
+	rm -rf libgowhisper.o build whisper

backend/go/whisper/gowhisper.cpp

@@ -107,7 +107,7 @@ int vad(float pcmf32[], size_t pcmf32_len, float **segs_out,
 }
 
 int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz,
-               float pcmf32[], size_t pcmf32_len, size_t *segs_out_len, char *prompt) {
+               float pcmf32[], size_t pcmf32_len, size_t *segs_out_len) {
   whisper_full_params wparams =
       whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
 
@@ -122,10 +122,8 @@ int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz,
   wparams.debug_mode = true;
   wparams.print_progress = true;
   wparams.tdrz_enable = tdrz;
-  wparams.initial_prompt = prompt;
 
   fprintf(stderr, "info: Enable tdrz: %d\n", tdrz);
-  fprintf(stderr, "info: Initial prompt: \"%s\"\n", prompt);
 
   if (whisper_full(ctx, wparams, pcmf32, pcmf32_len)) {
     fprintf(stderr, "error: transcription failed\n");

backend/go/whisper/gowhisper.go

@@ -17,7 +17,7 @@ var (
 	CppLoadModel                 func(modelPath string) int
 	CppLoadModelVAD              func(modelPath string) int
 	CppVAD                       func(pcmf32 []float32, pcmf32Size uintptr, segsOut unsafe.Pointer, segsOutLen unsafe.Pointer) int
-	CppTranscribe                func(threads uint32, lang string, translate bool, diarize bool, pcmf32 []float32, pcmf32Len uintptr, segsOutLen unsafe.Pointer, prompt string) int
+	CppTranscribe                func(threads uint32, lang string, translate bool, diarize bool, pcmf32 []float32, pcmf32Len uintptr, segsOutLen unsafe.Pointer) int
 	CppGetSegmentText            func(i int) string
 	CppGetSegmentStart           func(i int) int64
 	CppGetSegmentEnd             func(i int) int64
@@ -123,7 +123,7 @@ func (w *Whisper) AudioTranscription(opts *pb.TranscriptRequest) (pb.TranscriptR
 	segsLen := uintptr(0xdeadbeef)
 	segsLenPtr := unsafe.Pointer(&segsLen)
 
-	if ret := CppTranscribe(opts.Threads, opts.Language, opts.Translate, opts.Diarize, data, uintptr(len(data)), segsLenPtr, opts.Prompt); ret != 0 {
+	if ret := CppTranscribe(opts.Threads, opts.Language, opts.Translate, opts.Diarize, data, uintptr(len(data)), segsLenPtr); ret != 0 {
 		return pb.TranscriptResult{}, fmt.Errorf("Failed Transcribe")
 	}
 

backend/go/whisper/gowhisper.h

@@ -7,8 +7,7 @@ int load_model_vad(const char *const model_path);
 int vad(float pcmf32[], size_t pcmf32_size, float **segs_out,
         size_t *segs_out_len);
 int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz,
-               float pcmf32[], size_t pcmf32_len, size_t *segs_out_len,
-               char *prompt);
+               float pcmf32[], size_t pcmf32_len, size_t *segs_out_len);
 const char *get_segment_text(int i);
 int64_t get_segment_t0(int i);
 int64_t get_segment_t1(int i);

backend/go/whisper/main.go

@@ -3,7 +3,6 @@ package main
 // Note: this is started internally by LocalAI and a server is allocated for each model
 import (
 	"flag"
-	"os"
 
 	"github.com/ebitengine/purego"
 	grpc "github.com/mudler/LocalAI/pkg/grpc"
@@ -19,13 +18,7 @@ type LibFuncs struct {
 }
 
 func main() {
-	// Get library name from environment variable, default to fallback
-	libName := os.Getenv("WHISPER_LIBRARY")
-	if libName == "" {
-		libName = "./libgowhisper-fallback.so"
-	}
-
-	gosd, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+	gosd, err := purego.Dlopen("./libgowhisper.so", purego.RTLD_NOW|purego.RTLD_GLOBAL)
 	if err != nil {
 		panic(err)
 	}

backend/go/whisper/package.sh

@@ -10,8 +10,7 @@ CURDIR=$(dirname "$(realpath $0)")
 # Create lib directory
 mkdir -p $CURDIR/package/lib
 
-cp -avf $CURDIR/whisper $CURDIR/package/
-cp -fv $CURDIR/libgowhisper-*.so $CURDIR/package/
+cp -avf $CURDIR/whisper $CURDIR/libgowhisper.so $CURDIR/package/
 cp -fv $CURDIR/run.sh $CURDIR/package/
 
 # Detect architecture and copy appropriate libraries

backend/go/whisper/run.sh

@@ -1,52 +1,14 @@
 #!/bin/bash
 set -ex
 
-# Get the absolute current dir where the script is located
 CURDIR=$(dirname "$(realpath $0)")
 
-cd /
-
-echo "CPU info:"
-if [ "$(uname)" != "Darwin" ]; then
-	grep -e "model\sname" /proc/cpuinfo | head -1
-	grep -e "flags" /proc/cpuinfo | head -1
-fi
-
-LIBRARY="$CURDIR/libgowhisper-fallback.so"
-
-if [ "$(uname)" != "Darwin" ]; then
-	if grep -q -e "\savx\s" /proc/cpuinfo ; then
-		echo "CPU:    AVX    found OK"
-		if [ -e $CURDIR/libgowhisper-avx.so ]; then
-			LIBRARY="$CURDIR/libgowhisper-avx.so"
-		fi
-	fi
-
-	if grep -q -e "\savx2\s" /proc/cpuinfo ; then
-		echo "CPU:    AVX2   found OK"
-		if [ -e $CURDIR/libgowhisper-avx2.so ]; then
-			LIBRARY="$CURDIR/libgowhisper-avx2.so"
-		fi
-	fi
-
-	# Check avx 512
-	if grep -q -e "\savx512f\s" /proc/cpuinfo ; then
-		echo "CPU:    AVX512F found OK"
-		if [ -e $CURDIR/libgowhisper-avx512.so ]; then
-			LIBRARY="$CURDIR/libgowhisper-avx512.so"
-		fi
-	fi
-fi
-
 export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
-export WHISPER_LIBRARY=$LIBRARY
 
 # If there is a lib/ld.so, use it
 if [ -f $CURDIR/lib/ld.so ]; then
 	echo "Using lib/ld.so"
-	echo "Using library: $LIBRARY"
 	exec $CURDIR/lib/ld.so $CURDIR/whisper "$@"
 fi
 
-echo "Using library: $LIBRARY"
 exec $CURDIR/whisper "$@"

backend/index.yaml

@@ -25,10 +25,7 @@
     metal: "metal-llama-cpp"
     vulkan: "vulkan-llama-cpp"
     nvidia-l4t: "nvidia-l4t-arm64-llama-cpp"
-    nvidia-cuda-13: "cuda13-llama-cpp"
-    nvidia-cuda-12: "cuda12-llama-cpp"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-llama-cpp"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-llama-cpp"
+    darwin-x86: "darwin-x86-llama-cpp"
 - &whispercpp
   name: "whisper"
   alias: "whisper"
@@ -52,10 +49,6 @@
     amd: "rocm-whisper"
     vulkan: "vulkan-whisper"
     nvidia-l4t: "nvidia-l4t-arm64-whisper"
-    nvidia-cuda-13: "cuda13-whisper"
-    nvidia-cuda-12: "cuda12-whisper"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-whisper"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-whisper"
 - &stablediffusionggml
   name: "stablediffusion-ggml"
   alias: "stablediffusion-ggml"
@@ -80,10 +73,7 @@
     vulkan: "vulkan-stablediffusion-ggml"
     nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml"
     metal: "metal-stablediffusion-ggml"
-    nvidia-cuda-13: "cuda13-stablediffusion-ggml"
-    nvidia-cuda-12: "cuda12-stablediffusion-ggml"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-stablediffusion-ggml"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml"
+    # darwin-x86: "darwin-x86-stablediffusion-ggml"
 - &rfdetr
   name: "rfdetr"
   alias: "rfdetr"
@@ -106,9 +96,6 @@
     #amd: "rocm-rfdetr"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr"
     default: "cpu-rfdetr"
-    nvidia-cuda-13: "cuda13-rfdetr"
-    nvidia-cuda-12: "cuda12-rfdetr"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-rfdetr"
 - &vllm
   name: "vllm"
   license: apache-2.0
@@ -141,7 +128,6 @@
     nvidia: "cuda12-vllm"
     amd: "rocm-vllm"
     intel: "intel-vllm"
-    nvidia-cuda-12: "cuda12-vllm"
 - &mlx
   name: "mlx"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx"
@@ -215,8 +201,6 @@
     nvidia: "cuda12-transformers"
     intel: "intel-transformers"
     amd: "rocm-transformers"
-    nvidia-cuda-13: "cuda13-transformers"
-    nvidia-cuda-12: "cuda12-transformers"
 - &diffusers
   name: "diffusers"
   icon: https://raw.githubusercontent.com/huggingface/diffusers/main/docs/source/en/imgs/diffusers_library.jpg
@@ -237,10 +221,6 @@
     nvidia-l4t: "nvidia-l4t-diffusers"
     metal: "metal-diffusers"
     default: "cpu-diffusers"
-    nvidia-cuda-13: "cuda13-diffusers"
-    nvidia-cuda-12: "cuda12-diffusers"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-diffusers"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-diffusers"
 - &exllama2
   name: "exllama2"
   urls:
@@ -256,7 +236,6 @@
   capabilities:
     nvidia: "cuda12-exllama2"
     intel: "intel-exllama2"
-    nvidia-cuda-12: "cuda12-exllama2"
 - &faster-whisper
   icon: https://avatars.githubusercontent.com/u/1520500?s=200&v=4
   description: |
@@ -273,8 +252,6 @@
     nvidia: "cuda12-faster-whisper"
     intel: "intel-faster-whisper"
     amd: "rocm-faster-whisper"
-    nvidia-cuda-13: "cuda13-faster-whisper"
-    nvidia-cuda-12: "cuda12-faster-whisper"
 - &kokoro
   icon: https://avatars.githubusercontent.com/u/166769057?v=4
   description: |
@@ -294,9 +271,6 @@
     intel: "intel-kokoro"
     amd: "rocm-kokoro"
     nvidia-l4t: "nvidia-l4t-kokoro"
-    nvidia-cuda-13: "cuda13-kokoro"
-    nvidia-cuda-12: "cuda12-kokoro"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-kokoro"
 - &coqui
   urls:
     - https://github.com/idiap/coqui-ai-TTS
@@ -318,8 +292,6 @@
     nvidia: "cuda12-coqui"
     intel: "intel-coqui"
     amd: "rocm-coqui"
-    nvidia-cuda-13: "cuda13-coqui"
-    nvidia-cuda-12: "cuda12-coqui"
   icon: https://avatars.githubusercontent.com/u/1338804?s=200&v=4
 - &bark
   urls:
@@ -336,8 +308,6 @@
     cuda: "cuda12-bark"
     intel: "intel-bark"
     rocm: "rocm-bark"
-    nvidia-cuda-13: "cuda13-bark"
-    nvidia-cuda-12: "cuda12-bark"
   icon: https://avatars.githubusercontent.com/u/99442120?s=200&v=4
 - &barkcpp
   urls:
@@ -384,32 +354,6 @@
     metal: "metal-chatterbox"
     default: "cpu-chatterbox"
     nvidia-l4t: "nvidia-l4t-arm64-chatterbox"
-    nvidia-cuda-13: "cuda13-chatterbox"
-    nvidia-cuda-12: "cuda12-chatterbox"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-chatterbox"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-chatterbox"
-- &vibevoice
-  urls:
-    - https://github.com/microsoft/VibeVoice
-  description: |
-    VibeVoice-Realtime is a real-time text-to-speech model that generates natural-sounding speech.
-  tags:
-    - text-to-speech
-    - TTS
-  license: mit
-  name: "vibevoice"
-  alias: "vibevoice"
-  capabilities:
-    nvidia: "cuda12-vibevoice"
-    intel: "intel-vibevoice"
-    amd: "rocm-vibevoice"
-    nvidia-l4t: "nvidia-l4t-vibevoice"
-    default: "cpu-vibevoice"
-    nvidia-cuda-13: "cuda13-vibevoice"
-    nvidia-cuda-12: "cuda12-vibevoice"
-    nvidia-l4t-cuda-12: "nvidia-l4t-vibevoice"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vibevoice"
-  icon: https://avatars.githubusercontent.com/u/6154722?s=200&v=4
 - &piper
   name: "piper"
   uri: "quay.io/go-skynet/local-ai-backends:latest-piper"
@@ -483,86 +427,6 @@
     - text-to-speech
     - TTS
   license: apache-2.0
-- &neutts
-  name: "neutts"
-  urls:
-    - https://github.com/neuphonic/neutts-air
-  description: |
-    NeuTTS Air is the world’s first super-realistic, on-device, TTS speech language model with instant voice cloning. Built off a 0.5B LLM backbone, NeuTTS Air brings natural-sounding speech, real-time performance, built-in security and speaker cloning to your local device - unlocking a new category of embedded voice agents, assistants, toys, and compliance-safe apps.
-  tags:
-    - text-to-speech
-    - TTS
-  license: apache-2.0
-  capabilities:
-    default: "cpu-neutts"
-    nvidia: "cuda12-neutts"
-    amd: "rocm-neutts"
-    nvidia-l4t: "nvidia-l4t-neutts"
-    nvidia-cuda-12: "cuda12-neutts"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-neutts"
-- !!merge <<: *neutts
-  name: "neutts-development"
-  capabilities:
-    default: "cpu-neutts-development"
-    nvidia: "cuda12-neutts-development"
-    amd: "rocm-neutts-development"
-    nvidia-l4t: "nvidia-l4t-neutts-development"
-    nvidia-cuda-12: "cuda12-neutts-development"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-neutts-development"
-- !!merge <<: *llamacpp
-  name: "llama-cpp-development"
-  capabilities:
-    default: "cpu-llama-cpp-development"
-    nvidia: "cuda12-llama-cpp-development"
-    intel: "intel-sycl-f16-llama-cpp-development"
-    amd: "rocm-llama-cpp-development"
-    metal: "metal-llama-cpp-development"
-    vulkan: "vulkan-llama-cpp-development"
-    nvidia-l4t: "nvidia-l4t-arm64-llama-cpp-development"
-    nvidia-cuda-13: "cuda13-llama-cpp-development"
-    nvidia-cuda-12: "cuda12-llama-cpp-development"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-llama-cpp-development"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-llama-cpp-development"
-- !!merge <<: *neutts
-  name: "cpu-neutts"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts"
-  mirrors:
-    - localai/localai-backends:latest-cpu-neutts
-- !!merge <<: *neutts
-  name: "cuda12-neutts"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-neutts"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-12-neutts
-- !!merge <<: *neutts
-  name: "rocm-neutts"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-neutts"
-  mirrors:
-    - localai/localai-backends:latest-gpu-rocm-hipblas-neutts
-- !!merge <<: *neutts
-  name: "nvidia-l4t-arm64-neutts"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-neutts"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-arm64-neutts
-- !!merge <<: *neutts
-  name: "cpu-neutts-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-neutts"
-  mirrors:
-    - localai/localai-backends:master-cpu-neutts
-- !!merge <<: *neutts
-  name: "cuda12-neutts-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-neutts"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-12-neutts
-- !!merge <<: *neutts
-  name: "rocm-neutts-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-neutts"
-  mirrors:
-    - localai/localai-backends:master-gpu-rocm-hipblas-neutts
-- !!merge <<: *neutts
-  name: "nvidia-l4t-arm64-neutts-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-neutts"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-arm64-neutts
 - !!merge <<: *mlx
   name: "mlx-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx"
@@ -604,6 +468,16 @@
   mirrors:
     - localai/localai-backends:master-piper
 ## llama-cpp
+- !!merge <<: *llamacpp
+  name: "darwin-x86-llama-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-darwin-x86-llama-cpp"
+  mirrors:
+    - localai/localai-backends:latest-darwin-x86-llama-cpp
+- !!merge <<: *llamacpp
+  name: "darwin-x86-llama-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-darwin-x86-llama-cpp"
+  mirrors:
+    - localai/localai-backends:master-darwin-x86-llama-cpp
 - !!merge <<: *llamacpp
   name: "nvidia-l4t-arm64-llama-cpp"
   uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-llama-cpp"
@@ -614,16 +488,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-llama-cpp"
   mirrors:
     - localai/localai-backends:master-nvidia-l4t-arm64-llama-cpp
-- !!merge <<: *llamacpp
-  name: "cuda13-nvidia-l4t-arm64-llama-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-llama-cpp"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-llama-cpp
-- !!merge <<: *llamacpp
-  name: "cuda13-nvidia-l4t-arm64-llama-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-llama-cpp"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-llama-cpp
 - !!merge <<: *llamacpp
   name: "cpu-llama-cpp"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-llama-cpp"
@@ -704,16 +568,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-llama-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-intel-sycl-f16-llama-cpp
-- !!merge <<: *llamacpp
-  name: "cuda13-llama-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-llama-cpp
-- !!merge <<: *llamacpp
-  name: "cuda13-llama-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-llama-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-llama-cpp
 ## whisper
 - !!merge <<: *whispercpp
   name: "nvidia-l4t-arm64-whisper"
@@ -725,16 +579,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-whisper"
   mirrors:
     - localai/localai-backends:master-nvidia-l4t-arm64-whisper
-- !!merge <<: *whispercpp
-  name: "cuda13-nvidia-l4t-arm64-whisper"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-whisper"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-whisper
-- !!merge <<: *whispercpp
-  name: "cuda13-nvidia-l4t-arm64-whisper-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-whisper"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-whisper
 - !!merge <<: *whispercpp
   name: "cpu-whisper"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-whisper"
@@ -825,16 +669,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-whisper"
   mirrors:
     - localai/localai-backends:master-gpu-intel-sycl-f16-whisper
-- !!merge <<: *whispercpp
-  name: "cuda13-whisper"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-whisper"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-whisper
-- !!merge <<: *whispercpp
-  name: "cuda13-whisper-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-whisper"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-whisper
 ## stablediffusion-ggml
 - !!merge <<: *stablediffusionggml
   name: "cpu-stablediffusion-ggml"
@@ -914,26 +748,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-stablediffusion-ggml"
   mirrors:
     - localai/localai-backends:latest-nvidia-l4t-arm64-stablediffusion-ggml
-- !!merge <<: *stablediffusionggml
-  name: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-stablediffusion-ggml"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-stablediffusion-ggml
-- !!merge <<: *stablediffusionggml
-  name: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-stablediffusion-ggml"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-stablediffusion-ggml
-- !!merge <<: *stablediffusionggml
-  name: "cuda13-stablediffusion-ggml"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-stablediffusion-ggml"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-stablediffusion-ggml
-- !!merge <<: *stablediffusionggml
-  name: "cuda13-stablediffusion-ggml-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-stablediffusion-ggml"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-stablediffusion-ggml
 # vllm
 - !!merge <<: *vllm
   name: "vllm-development"
@@ -980,7 +794,6 @@
     #amd: "rocm-rfdetr-development"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr-development"
     default: "cpu-rfdetr-development"
-    nvidia-cuda-13: "cuda13-rfdetr-development"
 - !!merge <<: *rfdetr
   name: "cuda12-rfdetr"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-rfdetr"
@@ -1001,11 +814,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-rfdetr"
   mirrors:
     - localai/localai-backends:latest-nvidia-l4t-arm64-rfdetr
-- !!merge <<: *rfdetr
-  name: "nvidia-l4t-arm64-rfdetr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-rfdetr"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-arm64-rfdetr
 - !!merge <<: *rfdetr
   name: "cpu-rfdetr"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-rfdetr"
@@ -1036,16 +844,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-rfdetr"
   mirrors:
     - localai/localai-backends:latest-gpu-intel-rfdetr
-- !!merge <<: *rfdetr
-  name: "cuda13-rfdetr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-rfdetr"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-rfdetr
-- !!merge <<: *rfdetr
-  name: "cuda13-rfdetr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-rfdetr"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-rfdetr
 ## Rerankers
 - !!merge <<: *rerankers
   name: "rerankers-development"
@@ -1053,7 +851,6 @@
     nvidia: "cuda12-rerankers-development"
     intel: "intel-rerankers-development"
     amd: "rocm-rerankers-development"
-    nvidia-cuda-13: "cuda13-rerankers-development"
 - !!merge <<: *rerankers
   name: "cuda11-rerankers"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-11-rerankers"
@@ -1094,16 +891,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-rerankers"
   mirrors:
     - localai/localai-backends:master-gpu-intel-rerankers
-- !!merge <<: *rerankers
-  name: "cuda13-rerankers"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-rerankers"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-rerankers
-- !!merge <<: *rerankers
-  name: "cuda13-rerankers-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-rerankers"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-rerankers
 ## Transformers
 - !!merge <<: *transformers
   name: "transformers-development"
@@ -1111,7 +898,6 @@
     nvidia: "cuda12-transformers-development"
     intel: "intel-transformers-development"
     amd: "rocm-transformers-development"
-    nvidia-cuda-13: "cuda13-transformers-development"
 - !!merge <<: *transformers
   name: "cuda12-transformers"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-transformers"
@@ -1152,16 +938,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-transformers"
   mirrors:
     - localai/localai-backends:master-gpu-intel-transformers
-- !!merge <<: *transformers
-  name: "cuda13-transformers"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-transformers"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-transformers
-- !!merge <<: *transformers
-  name: "cuda13-transformers-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-transformers"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-transformers
 ## Diffusers
 - !!merge <<: *diffusers
   name: "diffusers-development"
@@ -1172,7 +948,6 @@
     nvidia-l4t: "nvidia-l4t-diffusers-development"
     metal: "metal-diffusers-development"
     default: "cpu-diffusers-development"
-    nvidia-cuda-13: "cuda13-diffusers-development"
 - !!merge <<: *diffusers
   name: "cpu-diffusers"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-diffusers"
@@ -1185,24 +960,14 @@
     - localai/localai-backends:master-cpu-diffusers
 - !!merge <<: *diffusers
   name: "nvidia-l4t-diffusers"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-diffusers"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-l4t-diffusers"
   mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-diffusers
+    - localai/localai-backends:latest-gpu-nvidia-l4t-diffusers
 - !!merge <<: *diffusers
   name: "nvidia-l4t-diffusers-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-diffusers"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-l4t-diffusers"
   mirrors:
-    - localai/localai-backends:master-nvidia-l4t-diffusers
-- !!merge <<: *diffusers
-  name: "cuda13-nvidia-l4t-arm64-diffusers"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-diffusers"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-diffusers
-- !!merge <<: *diffusers
-  name: "cuda13-nvidia-l4t-arm64-diffusers-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-diffusers"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-diffusers
+    - localai/localai-backends:master-gpu-nvidia-l4t-diffusers
 - !!merge <<: *diffusers
   name: "cuda12-diffusers"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-diffusers"
@@ -1243,16 +1008,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-diffusers"
   mirrors:
     - localai/localai-backends:master-gpu-intel-diffusers
-- !!merge <<: *diffusers
-  name: "cuda13-diffusers"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-diffusers"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-diffusers
-- !!merge <<: *diffusers
-  name: "cuda13-diffusers-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-diffusers"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-diffusers
 - !!merge <<: *diffusers
   name: "metal-diffusers"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-diffusers"
@@ -1324,14 +1079,14 @@
     - localai/localai-backends:master-gpu-intel-kokoro
 - !!merge <<: *kokoro
   name: "nvidia-l4t-kokoro"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-kokoro"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-l4t-kokoro"
   mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-kokoro
+    - localai/localai-backends:latest-gpu-nvidia-l4t-kokoro
 - !!merge <<: *kokoro
   name: "nvidia-l4t-kokoro-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-kokoro"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-l4t-kokoro"
   mirrors:
-    - localai/localai-backends:master-nvidia-l4t-kokoro
+    - localai/localai-backends:master-gpu-nvidia-l4t-kokoro
 - !!merge <<: *kokoro
   name: "cuda11-kokoro"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-11-kokoro"
@@ -1347,16 +1102,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-kokoro"
   mirrors:
     - localai/localai-backends:latest-gpu-rocm-hipblas-kokoro
-- !!merge <<: *kokoro
-  name: "cuda13-kokoro"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-kokoro"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-kokoro
-- !!merge <<: *kokoro
-  name: "cuda13-kokoro-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-kokoro"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-kokoro
 ## faster-whisper
 - !!merge <<: *faster-whisper
   name: "faster-whisper-development"
@@ -1364,7 +1109,6 @@
     nvidia: "cuda12-faster-whisper-development"
     intel: "intel-faster-whisper-development"
     amd: "rocm-faster-whisper-development"
-    nvidia-cuda-13: "cuda13-faster-whisper-development"
 - !!merge <<: *faster-whisper
   name: "cuda11-faster-whisper"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-11-faster-whisper"
@@ -1390,16 +1134,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-faster-whisper"
   mirrors:
     - localai/localai-backends:master-gpu-intel-faster-whisper
-- !!merge <<: *faster-whisper
-  name: "cuda13-faster-whisper"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-faster-whisper"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-faster-whisper
-- !!merge <<: *faster-whisper
-  name: "cuda13-faster-whisper-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-faster-whisper"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-faster-whisper
 ## coqui
 
 - !!merge <<: *coqui
@@ -1507,10 +1241,6 @@
     metal: "metal-chatterbox-development"
     default: "cpu-chatterbox-development"
     nvidia-l4t: "nvidia-l4t-arm64-chatterbox"
-    nvidia-cuda-13: "cuda13-chatterbox-development"
-    nvidia-cuda-12: "cuda12-chatterbox-development"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-chatterbox"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-chatterbox"
 - !!merge <<: *chatterbox
   name: "cpu-chatterbox"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-chatterbox"
@@ -1523,14 +1253,14 @@
     - localai/localai-backends:master-cpu-chatterbox
 - !!merge <<: *chatterbox
   name: "nvidia-l4t-arm64-chatterbox"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-chatterbox"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-l4t-arm64-chatterbox"
   mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-arm64-chatterbox
+    - localai/localai-backends:latest-gpu-nvidia-l4t-arm64-chatterbox
 - !!merge <<: *chatterbox
   name: "nvidia-l4t-arm64-chatterbox-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-chatterbox"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-l4t-arm64-chatterbox"
   mirrors:
-    - localai/localai-backends:master-nvidia-l4t-arm64-chatterbox
+    - localai/localai-backends:master-gpu-nvidia-l4t-arm64-chatterbox
 - !!merge <<: *chatterbox
   name: "metal-chatterbox"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-chatterbox"
@@ -1561,106 +1291,3 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-chatterbox"
   mirrors:
     - localai/localai-backends:latest-gpu-nvidia-cuda-12-chatterbox
-- !!merge <<: *chatterbox
-  name: "cuda13-chatterbox"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-chatterbox"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-chatterbox
-- !!merge <<: *chatterbox
-  name: "cuda13-chatterbox-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-chatterbox"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-chatterbox
-- !!merge <<: *chatterbox
-  name: "cuda13-nvidia-l4t-arm64-chatterbox"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-chatterbox"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-chatterbox
-- !!merge <<: *chatterbox
-  name: "cuda13-nvidia-l4t-arm64-chatterbox-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-chatterbox"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-chatterbox
-## vibevoice
-- !!merge <<: *vibevoice
-  name: "vibevoice-development"
-  capabilities:
-    nvidia: "cuda12-vibevoice-development"
-    intel: "intel-vibevoice-development"
-    amd: "rocm-vibevoice-development"
-    nvidia-l4t: "nvidia-l4t-vibevoice-development"
-    default: "cpu-vibevoice-development"
-    nvidia-cuda-13: "cuda13-vibevoice-development"
-    nvidia-cuda-12: "cuda12-vibevoice-development"
-    nvidia-l4t-cuda-12: "nvidia-l4t-vibevoice-development"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vibevoice-development"
-- !!merge <<: *vibevoice
-  name: "cpu-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-cpu-vibevoice
-- !!merge <<: *vibevoice
-  name: "cpu-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-cpu-vibevoice
-- !!merge <<: *vibevoice
-  name: "cuda12-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-12-vibevoice
-- !!merge <<: *vibevoice
-  name: "cuda12-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-12-vibevoice
-- !!merge <<: *vibevoice
-  name: "cuda13-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-vibevoice
-- !!merge <<: *vibevoice
-  name: "cuda13-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-vibevoice
-- !!merge <<: *vibevoice
-  name: "intel-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-gpu-intel-vibevoice
-- !!merge <<: *vibevoice
-  name: "intel-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-gpu-intel-vibevoice
-- !!merge <<: *vibevoice
-  name: "rocm-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-gpu-rocm-hipblas-vibevoice
-- !!merge <<: *vibevoice
-  name: "rocm-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-gpu-rocm-hipblas-vibevoice
-- !!merge <<: *vibevoice
-  name: "nvidia-l4t-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-vibevoice
-- !!merge <<: *vibevoice
-  name: "nvidia-l4t-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-vibevoice
-- !!merge <<: *vibevoice
-  name: "cuda13-nvidia-l4t-arm64-vibevoice"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-vibevoice"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-vibevoice
-- !!merge <<: *vibevoice
-  name: "cuda13-nvidia-l4t-arm64-vibevoice-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-vibevoice"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-vibevoice

backend/python/bark/requirements.txt

@@ -1,4 +1,4 @@
 bark==0.1.5
-grpcio==1.76.0
+grpcio==1.74.0
 protobuf
 certifi

backend/python/chatterbox/backend.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-This is an extra gRPC server of LocalAI for Chatterbox TTS
+This is an extra gRPC server of LocalAI for Bark TTS
 """
 from concurrent import futures
 import time
@@ -16,7 +16,6 @@ import torchaudio as ta
 from chatterbox.tts import ChatterboxTTS
 from chatterbox.mtl_tts import ChatterboxMultilingualTTS
 import grpc
-import tempfile
 
 def is_float(s):
     """Check if a string can be converted to float."""
@@ -33,79 +32,11 @@ def is_int(s):
     except ValueError:
         return False
 
-def split_text_at_word_boundary(text, max_length=250):
-    """
-    Split text at word boundaries without truncating words.
-    Returns a list of text chunks.
-    """
-    if not text or len(text) <= max_length:
-        return [text]
-    
-    chunks = []
-    words = text.split()
-    current_chunk = ""
-    
-    for word in words:
-        # Check if adding this word would exceed the limit
-        if len(current_chunk) + len(word) + 1 <= max_length:
-            if current_chunk:
-                current_chunk += " " + word
-            else:
-                current_chunk = word
-        else:
-            # If current chunk is not empty, add it to chunks
-            if current_chunk:
-                chunks.append(current_chunk)
-                current_chunk = word
-            else:
-                # If a single word is longer than max_length, we have to include it anyway
-                chunks.append(word)
-                current_chunk = ""
-    
-    # Add the last chunk if it's not empty
-    if current_chunk:
-        chunks.append(current_chunk)
-    
-    return chunks
-
-def merge_audio_files(audio_files, output_path, sample_rate):
-    """
-    Merge multiple audio files into a single audio file.
-    """
-    if not audio_files:
-        return
-    
-    if len(audio_files) == 1:
-        # If only one file, just copy it
-        import shutil
-        shutil.copy2(audio_files[0], output_path)
-        return
-    
-    # Load all audio files
-    waveforms = []
-    for audio_file in audio_files:
-        waveform, sr = ta.load(audio_file)
-        if sr != sample_rate:
-            # Resample if necessary
-            resampler = ta.transforms.Resample(sr, sample_rate)
-            waveform = resampler(waveform)
-        waveforms.append(waveform)
-    
-    # Concatenate all waveforms
-    merged_waveform = torch.cat(waveforms, dim=1)
-    
-    # Save the merged audio
-    ta.save(output_path, merged_waveform, sample_rate)
-    
-    # Clean up temporary files
-    for audio_file in audio_files:
-        if os.path.exists(audio_file):
-            os.remove(audio_file)
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 # If MAX_WORKERS are specified in the environment use it, otherwise default to 1
 MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+COQUI_LANGUAGE = os.environ.get('COQUI_LANGUAGE', None)
 
 # Implement the BackendServicer class with the service methods
 class BackendServicer(backend_pb2_grpc.BackendServicer):
@@ -187,33 +118,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # add options to kwargs
             kwargs.update(self.options)
 
-            # Check if text exceeds 250 characters
-            # (chatterbox does not support long text)
-            # https://github.com/resemble-ai/chatterbox/issues/60
-            # https://github.com/resemble-ai/chatterbox/issues/110
-            if len(request.text) > 250:
-                # Split text at word boundaries
-                text_chunks = split_text_at_word_boundary(request.text, max_length=250)
-                print(f"Splitting text into chunks of 250 characters: {len(text_chunks)}", file=sys.stderr)
-                # Generate audio for each chunk
-                temp_audio_files = []
-                for i, chunk in enumerate(text_chunks):
-                    # Generate audio for this chunk
-                    wav = self.model.generate(chunk, **kwargs)
-                    
-                    # Create temporary file for this chunk
-                    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.wav')
-                    temp_file.close()
-                    ta.save(temp_file.name, wav, self.model.sr)
-                    temp_audio_files.append(temp_file.name)
-                
-                # Merge all audio files
-                merge_audio_files(temp_audio_files, request.dst, self.model.sr)
-            else:
-                # Generate audio using ChatterboxTTS for short text
-                wav = self.model.generate(request.text, **kwargs)
-                # Save the generated audio
-                ta.save(request.dst, wav, self.model.sr)
+            # Generate audio using ChatterboxTTS
+            wav = self.model.generate(request.text, **kwargs)
+            # Save the generated audio
+            ta.save(request.dst, wav, self.model.sr)
             
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")

backend/python/chatterbox/requirements-cpu.txt

@@ -2,7 +2,6 @@
 accelerate
 torch
 torchaudio
-numpy>=1.24.0,<1.26.0
 transformers
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
 chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster

backend/python/chatterbox/requirements-cublas11.txt

@@ -2,7 +2,6 @@
 torch==2.6.0+cu118
 torchaudio==2.6.0+cu118
 transformers==4.46.3
-numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
 chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
 accelerate

backend/python/chatterbox/requirements-cublas12.txt

@@ -1,7 +1,6 @@
 torch
 torchaudio
 transformers
-numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
 chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
 accelerate

backend/python/chatterbox/requirements-cublas13.txt

@@ -1,8 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
-transformers
-numpy>=1.24.0,<1.26.0
-# https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
-chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
-accelerate

backend/python/chatterbox/requirements-hipblas.txt

@@ -2,7 +2,6 @@
 torch==2.6.0+rocm6.1
 torchaudio==2.6.0+rocm6.1
 transformers
-numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
 chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
 accelerate

backend/python/chatterbox/requirements-intel.txt

@@ -3,7 +3,6 @@ intel-extension-for-pytorch==2.3.110+xpu
 torch==2.3.1+cxx11.abi
 torchaudio==2.3.1+cxx11.abi
 transformers
-numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
 chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
 accelerate

backend/python/chatterbox/requirements-l4t.txt

@@ -2,6 +2,5 @@
 torch
 torchaudio
 transformers
-numpy>=1.24.0,<1.26.0
 chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
 accelerate

backend/python/chatterbox/requirements-l4t13.txt

@@ -1,7 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
-transformers
-numpy>=1.24.0,<1.26.0
-chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster
-accelerate

backend/python/chatterbox/requirements.txt

@@ -2,5 +2,4 @@ grpcio==1.71.0
 protobuf
 certifi
 packaging
-setuptools
-poetry
+setuptools

backend/python/common/libbackend.sh

@@ -211,7 +211,7 @@ function init() {
 # - hipblas
 # - intel
 function getBuildProfile() {
-    if [ x"${BUILD_TYPE:-}" == "xcublas" ] || [ x"${BUILD_TYPE:-}" == "xl4t" ]; then
+    if [ x"${BUILD_TYPE:-}" == "xcublas" ]; then
         if [ ! -z "${CUDA_MAJOR_VERSION:-}" ]; then
             echo ${BUILD_TYPE}${CUDA_MAJOR_VERSION}
         else
@@ -237,14 +237,7 @@ function getBuildProfile() {
 # Make the venv relocatable:
 # - rewrite venv/bin/python{,3} to relative symlinks into $(_portable_dir)
 # - normalize entrypoint shebangs to /usr/bin/env python3
-# - optionally update pyvenv.cfg to point to the portable Python directory (only at runtime)
-# Usage: _makeVenvPortable [--update-pyvenv-cfg]
 _makeVenvPortable() {
-    local update_pyvenv_cfg=false
-    if [ "${1:-}" = "--update-pyvenv-cfg" ]; then
-        update_pyvenv_cfg=true
-    fi
-
     local venv_dir="${EDIR}/venv"
     local vbin="${venv_dir}/bin"
 
@@ -262,39 +255,7 @@ _makeVenvPortable() {
     ln -s "${rel_py}" "${vbin}/python3"
     ln -s "python3" "${vbin}/python"
 
-    # 2) Update pyvenv.cfg to point to the portable Python directory (only at runtime)
-    #    Use absolute path resolved at runtime so it works when the venv is copied
-    if [ "$update_pyvenv_cfg" = "true" ]; then
-        local pyvenv_cfg="${venv_dir}/pyvenv.cfg"
-        if [ -f "${pyvenv_cfg}" ]; then
-            local portable_dir="$(_portable_dir)"
-            # Resolve to absolute path - this ensures it works when the backend is copied
-            # Only resolve if the directory exists (it should if ensurePortablePython was called)
-            if [ -d "${portable_dir}" ]; then
-                portable_dir="$(cd "${portable_dir}" && pwd)"
-            else
-                # Fallback to relative path if directory doesn't exist yet
-                portable_dir="../python"
-            fi
-            local sed_i=(sed -i)
-            # macOS/BSD sed needs a backup suffix; GNU sed doesn't. Make it portable:
-            if sed --version >/dev/null 2>&1; then
-                sed_i=(sed -i)
-            else
-                sed_i=(sed -i '')
-            fi
-            # Update the home field in pyvenv.cfg
-            # Handle both absolute paths (starting with /) and relative paths
-            if grep -q "^home = " "${pyvenv_cfg}"; then
-                "${sed_i[@]}" "s|^home = .*|home = ${portable_dir}|" "${pyvenv_cfg}"
-            else
-                # If home field doesn't exist, add it
-                echo "home = ${portable_dir}" >> "${pyvenv_cfg}"
-            fi
-        fi
-    fi
-
-    # 3) Rewrite shebangs of entry points to use env, so the venv is relocatable
+    # 2) Rewrite shebangs of entry points to use env, so the venv is relocatable
     #    Only touch text files that start with #! and reference the current venv.
     local ve_abs="${vbin}/python"
     local sed_i=(sed -i)
@@ -355,7 +316,6 @@ function ensureVenv() {
             fi
         fi
         if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then
-            # During install, only update symlinks and shebangs, not pyvenv.cfg
             _makeVenvPortable
         fi
     fi
@@ -460,11 +420,6 @@ function installRequirements() {
 #  - ${BACKEND_NAME}.py
 function startBackend() {
     ensureVenv
-    # Update pyvenv.cfg before running to ensure paths are correct for current location
-    # This is critical when the backend position is dynamic (e.g., copied from container)
-    if [ "x${PORTABLE_PYTHON}" == "xtrue" ] || [ -x "$(_portable_python)" ]; then
-        _makeVenvPortable --update-pyvenv-cfg
-    fi
     if [ ! -z "${BACKEND_FILE:-}" ]; then
         exec "${EDIR}/venv/bin/python" "${BACKEND_FILE}" "$@"
     elif [ -e "${MY_DIR}/server.py" ]; then

backend/python/common/template/requirements.txt

@@ -1,3 +1,3 @@
-grpcio==1.76.0
+grpcio==1.74.0
 protobuf
 grpcio-tools

backend/python/coqui/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.76.0
+grpcio==1.74.0
 protobuf
 certifi
 packaging==24.1

backend/python/diffusers/README.md

@@ -1,136 +1,5 @@
-# LocalAI Diffusers Backend
-
-This backend provides gRPC access to Hugging Face diffusers pipelines with dynamic pipeline loading.
-
-## Creating a separate environment for the diffusers project
+# Creating a separate environment for the diffusers project
 
 ```
 make diffusers
-```
-
-## Dynamic Pipeline Loader
-
-The diffusers backend includes a dynamic pipeline loader (`diffusers_dynamic_loader.py`) that automatically discovers and loads diffusers pipelines at runtime. This eliminates the need for per-pipeline conditional statements - new pipelines added to diffusers become available automatically without code changes.
-
-### How It Works
-
-1. **Pipeline Discovery**: On first use, the loader scans the `diffusers` package to find all classes that inherit from `DiffusionPipeline`.
-
-2. **Registry Caching**: Discovery results are cached for the lifetime of the process to avoid repeated scanning.
-
-3. **Task Aliases**: The loader automatically derives task aliases from class names (e.g., "text-to-image", "image-to-image", "inpainting") without hardcoding.
-
-4. **Multiple Resolution Methods**: Pipelines can be resolved by:
-   - Exact class name (e.g., `StableDiffusionPipeline`)
-   - Task alias (e.g., `text-to-image`, `img2img`)
-   - Model ID (uses HuggingFace Hub to infer pipeline type)
-
-### Usage Examples
-
-```python
-from diffusers_dynamic_loader import (
-    load_diffusers_pipeline,
-    get_available_pipelines,
-    get_available_tasks,
-    resolve_pipeline_class,
-    discover_diffusers_classes,
-    get_available_classes,
-)
-
-# List all available pipelines
-pipelines = get_available_pipelines()
-print(f"Available pipelines: {pipelines[:10]}...")
-
-# List all task aliases
-tasks = get_available_tasks()
-print(f"Available tasks: {tasks}")
-
-# Resolve a pipeline class by name
-cls = resolve_pipeline_class(class_name="StableDiffusionPipeline")
-
-# Resolve by task alias
-cls = resolve_pipeline_class(task="stable-diffusion")
-
-# Load and instantiate a pipeline
-pipe = load_diffusers_pipeline(
-    class_name="StableDiffusionPipeline",
-    model_id="runwayml/stable-diffusion-v1-5",
-    torch_dtype=torch.float16
-)
-
-# Load from single file
-pipe = load_diffusers_pipeline(
-    class_name="StableDiffusionPipeline",
-    model_id="/path/to/model.safetensors",
-    from_single_file=True,
-    torch_dtype=torch.float16
-)
-
-# Discover other diffusers classes (schedulers, models, etc.)
-schedulers = discover_diffusers_classes("SchedulerMixin")
-print(f"Available schedulers: {list(schedulers.keys())[:5]}...")
-
-# Get list of available scheduler classes
-scheduler_list = get_available_classes("SchedulerMixin")
-```
-
-### Generic Class Discovery
-
-The dynamic loader can discover not just pipelines but any class type from diffusers:
-
-```python
-# Discover all scheduler classes
-schedulers = discover_diffusers_classes("SchedulerMixin")
-
-# Discover all model classes
-models = discover_diffusers_classes("ModelMixin")
-
-# Get a sorted list of available classes
-scheduler_names = get_available_classes("SchedulerMixin")
-```
-
-### Special Pipeline Handling
-
-Most pipelines are loaded dynamically through `load_diffusers_pipeline()`. Only pipelines requiring truly custom initialization logic are handled explicitly:
-
-- `FluxTransformer2DModel`: Requires quantization and custom transformer loading (cannot use dynamic loader)
-- `WanPipeline` / `WanImageToVideoPipeline`: Uses dynamic loader with special VAE (float32 dtype)
-- `SanaPipeline`: Uses dynamic loader with post-load dtype conversion for VAE/text encoder
-- `StableVideoDiffusionPipeline`: Uses dynamic loader with CPU offload handling
-- `VideoDiffusionPipeline`: Alias for DiffusionPipeline with video flags
-
-All other pipelines (StableDiffusionPipeline, StableDiffusionXLPipeline, FluxPipeline, etc.) are loaded purely through the dynamic loader.
-
-### Error Handling
-
-When a pipeline cannot be resolved, the loader provides helpful error messages listing available pipelines and tasks:
-
-```
-ValueError: Unknown pipeline class 'NonExistentPipeline'. 
-Available pipelines: AnimateDiffPipeline, AnimateDiffVideoToVideoPipeline, ...
-```
-
-## Environment Variables
-
-| Variable | Default | Description |
-|----------|---------|-------------|
-| `COMPEL` | `0` | Enable Compel for prompt weighting |
-| `XPU` | `0` | Enable Intel XPU support |
-| `CLIPSKIP` | `1` | Enable CLIP skip support |
-| `SAFETENSORS` | `1` | Use safetensors format |
-| `CHUNK_SIZE` | `8` | Decode chunk size for video |
-| `FPS` | `7` | Video frames per second |
-| `DISABLE_CPU_OFFLOAD` | `0` | Disable CPU offload |
-| `FRAMES` | `64` | Number of video frames |
-| `BFL_REPO` | `ChuckMcSneed/FLUX.1-dev` | Flux base repo |
-| `PYTHON_GRPC_MAX_WORKERS` | `1` | Max gRPC workers |
-
-## Running Tests
-
-```bash
-./test.sh
-```
-
-The test suite includes:
-- Unit tests for the dynamic loader (`test_dynamic_loader.py`)
-- Integration tests for the gRPC backend (`test.py`)
+```

backend/python/diffusers/backend.py

@@ -1,10 +1,4 @@
 #!/usr/bin/env python3
-"""
-LocalAI Diffusers Backend
-
-This backend provides gRPC access to diffusers pipelines with dynamic pipeline loading.
-New pipelines added to diffusers become available automatically without code changes.
-"""
 from concurrent import futures
 import traceback
 import argparse
@@ -23,22 +17,14 @@ import backend_pb2_grpc
 
 import grpc
 
-# Import dynamic loader for pipeline discovery
-from diffusers_dynamic_loader import (
-    get_pipeline_registry,
-    resolve_pipeline_class,
-    get_available_pipelines,
-    load_diffusers_pipeline,
-)
-
-# Import specific items still needed for special cases and safety checker
-from diffusers import DiffusionPipeline, ControlNetModel
-from diffusers import FluxPipeline, FluxTransformer2DModel, AutoencoderKLWan
+from diffusers import SanaPipeline, StableDiffusion3Pipeline, StableDiffusionXLPipeline, StableDiffusionDepth2ImgPipeline, DPMSolverMultistepScheduler, StableDiffusionPipeline, DiffusionPipeline, \
+    EulerAncestralDiscreteScheduler, FluxPipeline, FluxTransformer2DModel, QwenImageEditPipeline, AutoencoderKLWan, WanPipeline, WanImageToVideoPipeline
+from diffusers import StableDiffusionImg2ImgPipeline, AutoPipelineForText2Image, ControlNetModel, StableVideoDiffusionPipeline, Lumina2Text2ImgPipeline
 from diffusers.pipelines.stable_diffusion import safety_checker
 from diffusers.utils import load_image, export_to_video
 from compel import Compel, ReturnedEmbeddingsType
 from optimum.quanto import freeze, qfloat8, quantize
-from transformers import T5EncoderModel
+from transformers import CLIPTextModel, T5EncoderModel
 from safetensors.torch import load_file
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
@@ -172,165 +158,6 @@ def get_scheduler(name: str, config: dict = {}):
 
 # Implement the BackendServicer class with the service methods
 class BackendServicer(backend_pb2_grpc.BackendServicer):
-
-    def _load_pipeline(self, request, modelFile, fromSingleFile, torchType, variant):
-        """
-        Load a diffusers pipeline dynamically using the dynamic loader.
-
-        This method uses load_diffusers_pipeline() for most pipelines, falling back
-        to explicit handling only for pipelines requiring custom initialization
-        (e.g., quantization, special VAE handling).
-
-        Args:
-            request: The gRPC request containing pipeline configuration
-            modelFile: Path to the model file (for single file loading)
-            fromSingleFile: Whether to use from_single_file() vs from_pretrained()
-            torchType: The torch dtype to use
-            variant: Model variant (e.g., "fp16")
-
-        Returns:
-            The loaded pipeline instance
-        """
-        pipeline_type = request.PipelineType
-
-        # Handle IMG2IMG request flag with default pipeline
-        if request.IMG2IMG and pipeline_type == "":
-            pipeline_type = "StableDiffusionImg2ImgPipeline"
-
-        # ================================================================
-        # Special cases requiring custom initialization logic
-        # Only handle pipelines that truly need custom code (quantization,
-        # special VAE handling, etc.). All other pipelines use dynamic loading.
-        # ================================================================
-
-        # FluxTransformer2DModel - requires quantization and custom transformer loading
-        if pipeline_type == "FluxTransformer2DModel":
-            dtype = torch.bfloat16
-            bfl_repo = os.environ.get("BFL_REPO", "ChuckMcSneed/FLUX.1-dev")
-
-            transformer = FluxTransformer2DModel.from_single_file(modelFile, torch_dtype=dtype)
-            quantize(transformer, weights=qfloat8)
-            freeze(transformer)
-            text_encoder_2 = T5EncoderModel.from_pretrained(bfl_repo, subfolder="text_encoder_2", torch_dtype=dtype)
-            quantize(text_encoder_2, weights=qfloat8)
-            freeze(text_encoder_2)
-
-            pipe = FluxPipeline.from_pretrained(bfl_repo, transformer=None, text_encoder_2=None, torch_dtype=dtype)
-            pipe.transformer = transformer
-            pipe.text_encoder_2 = text_encoder_2
-
-            if request.LowVRAM:
-                pipe.enable_model_cpu_offload()
-            return pipe
-
-        # WanPipeline - requires special VAE with float32 dtype
-        if pipeline_type == "WanPipeline":
-            vae = AutoencoderKLWan.from_pretrained(
-                request.Model,
-                subfolder="vae",
-                torch_dtype=torch.float32
-            )
-            pipe = load_diffusers_pipeline(
-                class_name="WanPipeline",
-                model_id=request.Model,
-                vae=vae,
-                torch_dtype=torchType
-            )
-            self.txt2vid = True
-            return pipe
-
-        # WanImageToVideoPipeline - requires special VAE with float32 dtype
-        if pipeline_type == "WanImageToVideoPipeline":
-            vae = AutoencoderKLWan.from_pretrained(
-                request.Model,
-                subfolder="vae",
-                torch_dtype=torch.float32
-            )
-            pipe = load_diffusers_pipeline(
-                class_name="WanImageToVideoPipeline",
-                model_id=request.Model,
-                vae=vae,
-                torch_dtype=torchType
-            )
-            self.img2vid = True
-            return pipe
-
-        # SanaPipeline - requires special VAE and text encoder dtype conversion
-        if pipeline_type == "SanaPipeline":
-            pipe = load_diffusers_pipeline(
-                class_name="SanaPipeline",
-                model_id=request.Model,
-                variant="bf16",
-                torch_dtype=torch.bfloat16
-            )
-            pipe.vae.to(torch.bfloat16)
-            pipe.text_encoder.to(torch.bfloat16)
-            return pipe
-
-        # VideoDiffusionPipeline - alias for DiffusionPipeline with txt2vid flag
-        if pipeline_type == "VideoDiffusionPipeline":
-            self.txt2vid = True
-            pipe = load_diffusers_pipeline(
-                class_name="DiffusionPipeline",
-                model_id=request.Model,
-                torch_dtype=torchType
-            )
-            return pipe
-
-        # StableVideoDiffusionPipeline - needs img2vid flag and CPU offload
-        if pipeline_type == "StableVideoDiffusionPipeline":
-            self.img2vid = True
-            pipe = load_diffusers_pipeline(
-                class_name="StableVideoDiffusionPipeline",
-                model_id=request.Model,
-                torch_dtype=torchType,
-                variant=variant
-            )
-            if not DISABLE_CPU_OFFLOAD:
-                pipe.enable_model_cpu_offload()
-            return pipe
-
-        # ================================================================
-        # Dynamic pipeline loading - the default path for most pipelines
-        # Uses the dynamic loader to instantiate any pipeline by class name
-        # ================================================================
-
-        # Build kwargs for dynamic loading
-        load_kwargs = {"torch_dtype": torchType}
-
-        # Add variant if not loading from single file
-        if not fromSingleFile and variant:
-            load_kwargs["variant"] = variant
-
-        # Add use_safetensors for from_pretrained
-        if not fromSingleFile:
-            load_kwargs["use_safetensors"] = SAFETENSORS
-
-        # Determine pipeline class name - default to AutoPipelineForText2Image
-        effective_pipeline_type = pipeline_type if pipeline_type else "AutoPipelineForText2Image"
-
-        # Use dynamic loader for all pipelines
-        try:
-            pipe = load_diffusers_pipeline(
-                class_name=effective_pipeline_type,
-                model_id=modelFile if fromSingleFile else request.Model,
-                from_single_file=fromSingleFile,
-                **load_kwargs
-            )
-        except Exception as e:
-            # Provide helpful error with available pipelines
-            available = get_available_pipelines()
-            raise ValueError(
-                f"Failed to load pipeline '{effective_pipeline_type}': {e}\n"
-                f"Available pipelines: {', '.join(available[:30])}..."
-            ) from e
-
-        # Apply LowVRAM optimization if supported and requested
-        if request.LowVRAM and hasattr(pipe, 'enable_model_cpu_offload'):
-            pipe.enable_model_cpu_offload()
-
-        return pipe
-
     def Health(self, request, context):
         return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
 
@@ -404,16 +231,139 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             fromSingleFile = request.Model.startswith("http") or request.Model.startswith("/") or local
             self.img2vid = False
             self.txt2vid = False
+            ## img2img
+            if (request.PipelineType == "StableDiffusionImg2ImgPipeline") or (request.IMG2IMG and request.PipelineType == ""):
+                if fromSingleFile:
+                    self.pipe = StableDiffusionImg2ImgPipeline.from_single_file(modelFile,
+                                                                                torch_dtype=torchType)
+                else:
+                    self.pipe = StableDiffusionImg2ImgPipeline.from_pretrained(request.Model,
+                                                                               torch_dtype=torchType)
 
-            # Load pipeline using dynamic loader
-            # Special cases that require custom initialization are handled first
-            self.pipe = self._load_pipeline(
-                request=request,
-                modelFile=modelFile,
-                fromSingleFile=fromSingleFile,
-                torchType=torchType,
-                variant=variant
-            )
+            elif request.PipelineType == "StableDiffusionDepth2ImgPipeline":
+                self.pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(request.Model,
+                                                                             torch_dtype=torchType)
+            ## img2vid
+            elif request.PipelineType == "StableVideoDiffusionPipeline":
+                self.img2vid = True
+                self.pipe = StableVideoDiffusionPipeline.from_pretrained(
+                    request.Model, torch_dtype=torchType, variant=variant
+                )
+                if not DISABLE_CPU_OFFLOAD:
+                    self.pipe.enable_model_cpu_offload()
+            ## text2img
+            elif request.PipelineType == "AutoPipelineForText2Image" or request.PipelineType == "":
+                self.pipe = AutoPipelineForText2Image.from_pretrained(request.Model,
+                                                                      torch_dtype=torchType,
+                                                                      use_safetensors=SAFETENSORS,
+                                                                      variant=variant)
+            elif request.PipelineType == "StableDiffusionPipeline":
+                if fromSingleFile:
+                    self.pipe = StableDiffusionPipeline.from_single_file(modelFile,
+                                                                         torch_dtype=torchType)
+                else:
+                    self.pipe = StableDiffusionPipeline.from_pretrained(request.Model,
+                                                                        torch_dtype=torchType)
+            elif request.PipelineType == "DiffusionPipeline":
+                self.pipe = DiffusionPipeline.from_pretrained(request.Model,
+                                                              torch_dtype=torchType)
+            elif request.PipelineType == "QwenImageEditPipeline":
+                self.pipe = QwenImageEditPipeline.from_pretrained(request.Model,
+                                                                 torch_dtype=torchType)
+            elif request.PipelineType == "VideoDiffusionPipeline":
+                self.txt2vid = True
+                self.pipe = DiffusionPipeline.from_pretrained(request.Model,
+                                                              torch_dtype=torchType)
+            elif request.PipelineType == "StableDiffusionXLPipeline":
+                if fromSingleFile:
+                    self.pipe = StableDiffusionXLPipeline.from_single_file(modelFile,
+                                                                           torch_dtype=torchType,
+                                                                           use_safetensors=True)
+                else:
+                    self.pipe = StableDiffusionXLPipeline.from_pretrained(
+                        request.Model,
+                        torch_dtype=torchType,
+                        use_safetensors=True,
+                        variant=variant)
+            elif request.PipelineType == "StableDiffusion3Pipeline":
+                if fromSingleFile:
+                    self.pipe = StableDiffusion3Pipeline.from_single_file(modelFile,
+                                                                          torch_dtype=torchType,
+                                                                          use_safetensors=True)
+                else:
+                    self.pipe = StableDiffusion3Pipeline.from_pretrained(
+                        request.Model,
+                        torch_dtype=torchType,
+                        use_safetensors=True,
+                        variant=variant)
+            elif request.PipelineType == "FluxPipeline":
+                if fromSingleFile:
+                    self.pipe = FluxPipeline.from_single_file(modelFile,
+                                                              torch_dtype=torchType,
+                                                              use_safetensors=True)
+                else:
+                    self.pipe = FluxPipeline.from_pretrained(
+                        request.Model,
+                        torch_dtype=torch.bfloat16)
+                if request.LowVRAM:
+                    self.pipe.enable_model_cpu_offload()
+            elif request.PipelineType == "FluxTransformer2DModel":
+                    dtype = torch.bfloat16
+                    # specify from environment or default to "ChuckMcSneed/FLUX.1-dev"
+                    bfl_repo = os.environ.get("BFL_REPO", "ChuckMcSneed/FLUX.1-dev")
+
+                    transformer = FluxTransformer2DModel.from_single_file(modelFile, torch_dtype=dtype)
+                    quantize(transformer, weights=qfloat8)
+                    freeze(transformer)
+                    text_encoder_2 = T5EncoderModel.from_pretrained(bfl_repo, subfolder="text_encoder_2", torch_dtype=dtype)
+                    quantize(text_encoder_2, weights=qfloat8)
+                    freeze(text_encoder_2)
+
+                    self.pipe = FluxPipeline.from_pretrained(bfl_repo, transformer=None, text_encoder_2=None, torch_dtype=dtype)
+                    self.pipe.transformer = transformer
+                    self.pipe.text_encoder_2 = text_encoder_2
+
+                    if request.LowVRAM:
+                        self.pipe.enable_model_cpu_offload()
+            elif request.PipelineType == "Lumina2Text2ImgPipeline":
+                self.pipe = Lumina2Text2ImgPipeline.from_pretrained(
+                    request.Model,
+                    torch_dtype=torch.bfloat16)
+                if request.LowVRAM:
+                    self.pipe.enable_model_cpu_offload()
+            elif request.PipelineType == "SanaPipeline":
+                self.pipe = SanaPipeline.from_pretrained(
+                    request.Model,
+                    variant="bf16",
+                    torch_dtype=torch.bfloat16)
+                self.pipe.vae.to(torch.bfloat16)
+                self.pipe.text_encoder.to(torch.bfloat16)
+            elif request.PipelineType == "WanPipeline":
+                # WAN2.2 pipeline requires special VAE handling
+                vae = AutoencoderKLWan.from_pretrained(
+                    request.Model, 
+                    subfolder="vae", 
+                    torch_dtype=torch.float32
+                )
+                self.pipe = WanPipeline.from_pretrained(
+                    request.Model,
+                    vae=vae,
+                    torch_dtype=torchType
+                )
+                self.txt2vid = True  # WAN2.2 is a text-to-video pipeline
+            elif request.PipelineType == "WanImageToVideoPipeline":
+                # WAN2.2 image-to-video pipeline
+                vae = AutoencoderKLWan.from_pretrained(
+                    request.Model, 
+                    subfolder="vae", 
+                    torch_dtype=torch.float32
+                )
+                self.pipe = WanImageToVideoPipeline.from_pretrained(
+                    request.Model,
+                    vae=vae,
+                    torch_dtype=torchType
+                )
+                self.img2vid = True  # WAN2.2 image-to-video pipeline
 
             if CLIPSKIP and request.CLIPSkip != 0:
                 self.clip_skip = request.CLIPSkip
@@ -551,12 +501,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # create a dictionary of values for the parameters
         options = {
+            "negative_prompt": request.negative_prompt,
             "num_inference_steps": steps,
         }
 
-        if hasattr(request, 'negative_prompt') and request.negative_prompt != "":
-            options["negative_prompt"] = request.negative_prompt
-
         # Handle image source: prioritize RefImages over request.src
         image_src = None
         if hasattr(request, 'ref_images') and request.ref_images and len(request.ref_images) > 0:
@@ -580,7 +528,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         if CLIPSKIP and self.clip_skip != 0:
             options["clip_skip"] = self.clip_skip
 
-        kwargs = {}
+        # Get the keys that we will build the args for our pipe for
+        keys = options.keys()
+
+        if request.EnableParameters != "":
+            keys = [key.strip() for key in request.EnableParameters.split(",")]
+
+        if request.EnableParameters == "none":
+            keys = []
+
+        # create a dictionary of parameters by using the keys from EnableParameters and the values from defaults
+        kwargs = {key: options.get(key) for key in keys if key in options}
 
         # populate kwargs from self.options.
         kwargs.update(self.options)

backend/python/diffusers/diffusers_dynamic_loader.py

@@ -1,538 +0,0 @@
-"""
-Dynamic Diffusers Pipeline Loader
-
-This module provides dynamic discovery and loading of diffusers pipelines at runtime,
-eliminating the need for per-pipeline conditional statements. New pipelines added to
-diffusers become available automatically without code changes.
-
-The module also supports discovering other diffusers classes like schedulers, models,
-and other components, making it a generic solution for dynamic class loading.
-
-Usage:
-    from diffusers_dynamic_loader import load_diffusers_pipeline, get_available_pipelines
-
-    # Load by class name
-    pipe = load_diffusers_pipeline(class_name="StableDiffusionPipeline", model_id="...", torch_dtype=torch.float16)
-
-    # Load by task alias
-    pipe = load_diffusers_pipeline(task="text-to-image", model_id="...", torch_dtype=torch.float16)
-
-    # Load using model_id (infers from HuggingFace Hub if possible)
-    pipe = load_diffusers_pipeline(model_id="runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
-
-    # Get list of available pipelines
-    available = get_available_pipelines()
-
-    # Discover other diffusers classes (schedulers, models, etc.)
-    schedulers = discover_diffusers_classes("SchedulerMixin")
-    models = discover_diffusers_classes("ModelMixin")
-"""
-
-import importlib
-import re
-import sys
-from typing import Any, Dict, List, Optional, Tuple, Type
-
-
-# Global cache for discovered pipelines - computed once per process
-_pipeline_registry: Optional[Dict[str, Type]] = None
-_task_aliases: Optional[Dict[str, List[str]]] = None
-
-# Global cache for other discovered class types
-_class_registries: Dict[str, Dict[str, Type]] = {}
-
-
-def _camel_to_kebab(name: str) -> str:
-    """
-    Convert CamelCase to kebab-case.
-
-    Examples:
-        StableDiffusionPipeline -> stable-diffusion-pipeline
-        StableDiffusionXLImg2ImgPipeline -> stable-diffusion-xl-img-2-img-pipeline
-    """
-    # Insert hyphen before uppercase letters (but not at the start)
-    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', name)
-    # Insert hyphen before uppercase letters following lowercase letters or numbers
-    s2 = re.sub('([a-z0-9])([A-Z])', r'\1-\2', s1)
-    return s2.lower()
-
-
-def _extract_task_keywords(class_name: str) -> List[str]:
-    """
-    Extract task-related keywords from a pipeline class name.
-
-    This function derives useful task aliases from the class name without
-    hardcoding per-pipeline branches.
-
-    Returns a list of potential task aliases for this pipeline.
-    """
-    aliases = []
-    name_lower = class_name.lower()
-
-    # Direct task mappings based on common patterns in class names
-    task_patterns = {
-        'text2image': ['text-to-image', 'txt2img', 'text2image'],
-        'texttoimage': ['text-to-image', 'txt2img', 'text2image'],
-        'txt2img': ['text-to-image', 'txt2img', 'text2image'],
-        'img2img': ['image-to-image', 'img2img', 'image2image'],
-        'image2image': ['image-to-image', 'img2img', 'image2image'],
-        'imagetoimage': ['image-to-image', 'img2img', 'image2image'],
-        'img2video': ['image-to-video', 'img2vid', 'img2video'],
-        'imagetovideo': ['image-to-video', 'img2vid', 'img2video'],
-        'text2video': ['text-to-video', 'txt2vid', 'text2video'],
-        'texttovideo': ['text-to-video', 'txt2vid', 'text2video'],
-        'inpaint': ['inpainting', 'inpaint'],
-        'depth2img': ['depth-to-image', 'depth2img'],
-        'depthtoimage': ['depth-to-image', 'depth2img'],
-        'controlnet': ['controlnet', 'control-net'],
-        'upscale': ['upscaling', 'upscale', 'super-resolution'],
-        'superresolution': ['upscaling', 'upscale', 'super-resolution'],
-    }
-
-    # Check for each pattern in the class name
-    for pattern, task_aliases in task_patterns.items():
-        if pattern in name_lower:
-            aliases.extend(task_aliases)
-
-    # Also detect general pipeline types from the class name structure
-    # E.g., StableDiffusionPipeline -> stable-diffusion, flux -> flux
-    # Remove "Pipeline" suffix and convert to kebab case
-    if class_name.endswith('Pipeline'):
-        base_name = class_name[:-8]  # Remove "Pipeline"
-        kebab_name = _camel_to_kebab(base_name)
-        aliases.append(kebab_name)
-
-        # Extract model family name (e.g., "stable-diffusion" from "stable-diffusion-xl-img-2-img")
-        parts = kebab_name.split('-')
-        if len(parts) >= 2:
-            # Try the first two words as a family name
-            family = '-'.join(parts[:2])
-            if family not in aliases:
-                aliases.append(family)
-
-    # If no specific task pattern matched but class contains "Pipeline", add "text-to-image" as default
-    # since most diffusion pipelines support text-to-image generation
-    if 'text-to-image' not in aliases and 'image-to-image' not in aliases:
-        # Only add for pipelines that seem to be generation pipelines (not schedulers, etc.)
-        if 'pipeline' in name_lower and not any(x in name_lower for x in ['scheduler', 'processor', 'encoder']):
-            # Don't automatically add - let it be explicit
-            pass
-
-    return list(set(aliases))  # Remove duplicates
-
-
-def discover_diffusers_classes(
-    base_class_name: str,
-    include_base: bool = True
-) -> Dict[str, Type]:
-    """
-    Discover all subclasses of a given base class from diffusers.
-
-    This function provides a generic way to discover any type of diffusers class,
-    not just pipelines. It can be used to discover schedulers, models, processors,
-    and other components.
-
-    Args:
-        base_class_name: Name of the base class to search for subclasses
-                        (e.g., "DiffusionPipeline", "SchedulerMixin", "ModelMixin")
-        include_base: Whether to include the base class itself in results
-
-    Returns:
-        Dict mapping class names to class objects
-
-    Examples:
-        # Discover all pipeline classes
-        pipelines = discover_diffusers_classes("DiffusionPipeline")
-
-        # Discover all scheduler classes
-        schedulers = discover_diffusers_classes("SchedulerMixin")
-
-        # Discover all model classes
-        models = discover_diffusers_classes("ModelMixin")
-
-        # Discover AutoPipeline classes
-        auto_pipelines = discover_diffusers_classes("AutoPipelineForText2Image")
-    """
-    global _class_registries
-
-    # Check cache first
-    if base_class_name in _class_registries:
-        return _class_registries[base_class_name]
-
-    import diffusers
-
-    # Try to get the base class from diffusers
-    base_class = None
-    try:
-        base_class = getattr(diffusers, base_class_name)
-    except AttributeError:
-        # Try to find in submodules
-        for submodule in ['schedulers', 'models', 'pipelines']:
-            try:
-                module = importlib.import_module(f'diffusers.{submodule}')
-                if hasattr(module, base_class_name):
-                    base_class = getattr(module, base_class_name)
-                    break
-            except (ImportError, ModuleNotFoundError):
-                continue
-
-    if base_class is None:
-        raise ValueError(f"Could not find base class '{base_class_name}' in diffusers")
-
-    registry: Dict[str, Type] = {}
-
-    # Include base class if requested
-    if include_base:
-        registry[base_class_name] = base_class
-
-    # Scan diffusers module for subclasses
-    for attr_name in dir(diffusers):
-        try:
-            attr = getattr(diffusers, attr_name)
-            if (isinstance(attr, type) and
-                issubclass(attr, base_class) and
-                (include_base or attr is not base_class)):
-                registry[attr_name] = attr
-        except (ImportError, AttributeError, TypeError, RuntimeError, ModuleNotFoundError):
-            continue
-
-    # Cache the results
-    _class_registries[base_class_name] = registry
-    return registry
-
-
-def get_available_classes(base_class_name: str) -> List[str]:
-    """
-    Get a sorted list of all discovered class names for a given base class.
-
-    Args:
-        base_class_name: Name of the base class (e.g., "SchedulerMixin")
-
-    Returns:
-        Sorted list of discovered class names
-    """
-    return sorted(discover_diffusers_classes(base_class_name).keys())
-
-
-def _discover_pipelines() -> Tuple[Dict[str, Type], Dict[str, List[str]]]:
-    """
-    Discover all subclasses of DiffusionPipeline from diffusers.
-
-    This function uses the generic discover_diffusers_classes() internally
-    and adds pipeline-specific task alias generation. It also includes
-    AutoPipeline classes which are special utility classes for automatic
-    pipeline selection.
-
-    Returns:
-        A tuple of (pipeline_registry, task_aliases) where:
-        - pipeline_registry: Dict mapping class names to class objects
-        - task_aliases: Dict mapping task aliases to lists of class names
-    """
-    # Use the generic discovery function
-    pipeline_registry = discover_diffusers_classes("DiffusionPipeline", include_base=True)
-
-    # Also add AutoPipeline classes - these are special utility classes that are
-    # NOT subclasses of DiffusionPipeline but are commonly used
-    import diffusers
-    auto_pipeline_classes = [
-        "AutoPipelineForText2Image",
-        "AutoPipelineForImage2Image",
-        "AutoPipelineForInpainting",
-    ]
-    for cls_name in auto_pipeline_classes:
-        try:
-            cls = getattr(diffusers, cls_name)
-            if cls is not None:
-                pipeline_registry[cls_name] = cls
-        except AttributeError:
-            # Class not available in this version of diffusers
-            pass
-
-    # Generate task aliases for pipelines
-    task_aliases: Dict[str, List[str]] = {}
-    for attr_name in pipeline_registry:
-        if attr_name == "DiffusionPipeline":
-            continue  # Skip base class for alias generation
-
-        aliases = _extract_task_keywords(attr_name)
-        for alias in aliases:
-            if alias not in task_aliases:
-                task_aliases[alias] = []
-            if attr_name not in task_aliases[alias]:
-                task_aliases[alias].append(attr_name)
-
-    return pipeline_registry, task_aliases
-
-
-def get_pipeline_registry() -> Dict[str, Type]:
-    """
-    Get the cached pipeline registry.
-
-    Returns a dictionary mapping pipeline class names to their class objects.
-    The registry is built on first access and cached for subsequent calls.
-    """
-    global _pipeline_registry, _task_aliases
-    if _pipeline_registry is None:
-        _pipeline_registry, _task_aliases = _discover_pipelines()
-    return _pipeline_registry
-
-
-def get_task_aliases() -> Dict[str, List[str]]:
-    """
-    Get the cached task aliases dictionary.
-
-    Returns a dictionary mapping task aliases (e.g., "text-to-image") to
-    lists of pipeline class names that support that task.
-    """
-    global _pipeline_registry, _task_aliases
-    if _task_aliases is None:
-        _pipeline_registry, _task_aliases = _discover_pipelines()
-    return _task_aliases
-
-
-def get_available_pipelines() -> List[str]:
-    """
-    Get a sorted list of all discovered pipeline class names.
-
-    Returns:
-        List of pipeline class names available for loading.
-    """
-    return sorted(get_pipeline_registry().keys())
-
-
-def get_available_tasks() -> List[str]:
-    """
-    Get a sorted list of all available task aliases.
-
-    Returns:
-        List of task aliases (e.g., ["text-to-image", "image-to-image", ...])
-    """
-    return sorted(get_task_aliases().keys())
-
-
-def resolve_pipeline_class(
-    class_name: Optional[str] = None,
-    task: Optional[str] = None,
-    model_id: Optional[str] = None
-) -> Type:
-    """
-    Resolve a pipeline class from class_name, task, or model_id.
-
-    Priority:
-    1. If class_name is provided, look it up directly
-    2. If task is provided, resolve through task aliases
-    3. If model_id is provided, try to infer from HuggingFace Hub
-
-    Args:
-        class_name: Exact pipeline class name (e.g., "StableDiffusionPipeline")
-        task: Task alias (e.g., "text-to-image", "img2img")
-        model_id: HuggingFace model ID (e.g., "runwayml/stable-diffusion-v1-5")
-
-    Returns:
-        The resolved pipeline class.
-
-    Raises:
-        ValueError: If no pipeline could be resolved.
-    """
-    registry = get_pipeline_registry()
-    aliases = get_task_aliases()
-
-    # 1. Direct class name lookup
-    if class_name:
-        if class_name in registry:
-            return registry[class_name]
-        # Try case-insensitive match
-        for name, cls in registry.items():
-            if name.lower() == class_name.lower():
-                return cls
-        raise ValueError(
-            f"Unknown pipeline class '{class_name}'. "
-            f"Available pipelines: {', '.join(sorted(registry.keys())[:20])}..."
-        )
-
-    # 2. Task alias lookup
-    if task:
-        task_lower = task.lower().replace('_', '-')
-        if task_lower in aliases:
-            # Return the first matching pipeline for this task
-            matching_classes = aliases[task_lower]
-            if matching_classes:
-                return registry[matching_classes[0]]
-
-        # Try partial matching
-        for alias, classes in aliases.items():
-            if task_lower in alias or alias in task_lower:
-                if classes:
-                    return registry[classes[0]]
-
-        raise ValueError(
-            f"Unknown task '{task}'. "
-            f"Available tasks: {', '.join(sorted(aliases.keys())[:20])}..."
-        )
-
-    # 3. Try to infer from HuggingFace Hub
-    if model_id:
-        try:
-            from huggingface_hub import model_info
-            info = model_info(model_id)
-
-            # Check pipeline_tag
-            if hasattr(info, 'pipeline_tag') and info.pipeline_tag:
-                tag = info.pipeline_tag.lower().replace('_', '-')
-                if tag in aliases:
-                    matching_classes = aliases[tag]
-                    if matching_classes:
-                        return registry[matching_classes[0]]
-
-            # Check model card for hints
-            if hasattr(info, 'cardData') and info.cardData:
-                card = info.cardData
-                if 'pipeline_tag' in card:
-                    tag = card['pipeline_tag'].lower().replace('_', '-')
-                    if tag in aliases:
-                        matching_classes = aliases[tag]
-                        if matching_classes:
-                            return registry[matching_classes[0]]
-
-        except ImportError:
-            # huggingface_hub not available
-            pass
-        except (KeyError, AttributeError, ValueError, OSError):
-            # Model info lookup failed - common cases:
-            # - KeyError: Missing keys in model card
-            # - AttributeError: Missing attributes on model info
-            # - ValueError: Invalid model data
-            # - OSError: Network or file access issues
-            pass
-
-        # Fallback: use DiffusionPipeline.from_pretrained which auto-detects
-        # DiffusionPipeline is always added to registry in _discover_pipelines (line 132)
-        # but use .get() with import fallback for extra safety
-        from diffusers import DiffusionPipeline
-        return registry.get('DiffusionPipeline', DiffusionPipeline)
-
-    raise ValueError(
-        "Must provide at least one of: class_name, task, or model_id. "
-        f"Available pipelines: {', '.join(sorted(registry.keys())[:20])}... "
-        f"Available tasks: {', '.join(sorted(aliases.keys())[:20])}..."
-    )
-
-
-def load_diffusers_pipeline(
-    class_name: Optional[str] = None,
-    task: Optional[str] = None,
-    model_id: Optional[str] = None,
-    from_single_file: bool = False,
-    **kwargs
-) -> Any:
-    """
-    Load a diffusers pipeline dynamically.
-
-    This function resolves the appropriate pipeline class based on the provided
-    parameters and instantiates it with the given kwargs.
-
-    Args:
-        class_name: Exact pipeline class name (e.g., "StableDiffusionPipeline")
-        task: Task alias (e.g., "text-to-image", "img2img")
-        model_id: HuggingFace model ID or local path
-        from_single_file: If True, use from_single_file() instead of from_pretrained()
-        **kwargs: Additional arguments passed to from_pretrained() or from_single_file()
-
-    Returns:
-        An instantiated pipeline object.
-
-    Raises:
-        ValueError: If no pipeline could be resolved.
-        Exception: If pipeline loading fails.
-
-    Examples:
-        # Load by class name
-        pipe = load_diffusers_pipeline(
-            class_name="StableDiffusionPipeline",
-            model_id="runwayml/stable-diffusion-v1-5",
-            torch_dtype=torch.float16
-        )
-
-        # Load by task
-        pipe = load_diffusers_pipeline(
-            task="text-to-image",
-            model_id="runwayml/stable-diffusion-v1-5",
-            torch_dtype=torch.float16
-        )
-
-        # Load from single file
-        pipe = load_diffusers_pipeline(
-            class_name="StableDiffusionPipeline",
-            model_id="/path/to/model.safetensors",
-            from_single_file=True,
-            torch_dtype=torch.float16
-        )
-    """
-    # Resolve the pipeline class
-    pipeline_class = resolve_pipeline_class(
-        class_name=class_name,
-        task=task,
-        model_id=model_id
-    )
-
-    # If no model_id provided but we have a class, we can't load
-    if model_id is None:
-        raise ValueError("model_id is required to load a pipeline")
-
-    # Load the pipeline
-    try:
-        if from_single_file:
-            # Check if the class has from_single_file method
-            if hasattr(pipeline_class, 'from_single_file'):
-                return pipeline_class.from_single_file(model_id, **kwargs)
-            else:
-                raise ValueError(
-                    f"Pipeline class {pipeline_class.__name__} does not support from_single_file(). "
-                    f"Use from_pretrained() instead."
-                )
-        else:
-            return pipeline_class.from_pretrained(model_id, **kwargs)
-
-    except Exception as e:
-        # Provide helpful error message
-        available = get_available_pipelines()
-        raise RuntimeError(
-            f"Failed to load pipeline '{pipeline_class.__name__}' from '{model_id}': {e}\n"
-            f"Available pipelines: {', '.join(available[:20])}..."
-        ) from e
-
-
-def get_pipeline_info(class_name: str) -> Dict[str, Any]:
-    """
-    Get information about a specific pipeline class.
-
-    Args:
-        class_name: The pipeline class name
-
-    Returns:
-        Dictionary with pipeline information including:
-        - name: Class name
-        - aliases: List of task aliases
-        - supports_single_file: Whether from_single_file() is available
-        - docstring: Class docstring (if available)
-    """
-    registry = get_pipeline_registry()
-    aliases = get_task_aliases()
-
-    if class_name not in registry:
-        raise ValueError(f"Unknown pipeline: {class_name}")
-
-    cls = registry[class_name]
-
-    # Find all aliases for this pipeline
-    pipeline_aliases = []
-    for alias, classes in aliases.items():
-        if class_name in classes:
-            pipeline_aliases.append(alias)
-
-    return {
-        'name': class_name,
-        'aliases': pipeline_aliases,
-        'supports_single_file': hasattr(cls, 'from_single_file'),
-        'docstring': cls.__doc__[:200] if cls.__doc__ else None
-    }

backend/python/diffusers/install.sh

@@ -16,11 +16,4 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then
     EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
 fi
 
-# Use python 3.12 for l4t
-if [ "x${BUILD_PROFILE}" == "xl4t12" ] || [ "x${BUILD_PROFILE}" == "xl4t13" ]; then
-  PYTHON_VERSION="3.12"
-  PYTHON_PATCH="12"
-  PY_STANDALONE_TAG="20251120"
-fi
-
 installRequirements

backend/python/diffusers/requirements-cublas13.txt

@@ -1,12 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-git+https://github.com/huggingface/diffusers
-opencv-python
-transformers
-torchvision
-accelerate
-compel
-peft
-sentencepiece
-torch
-ftfy
-optimum-quanto

backend/python/diffusers/requirements-l4t.txt

@@ -0,0 +1,12 @@
+--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu126/
+torch
+diffusers
+transformers
+accelerate
+compel
+peft
+optimum-quanto
+numpy<2
+sentencepiece
+torchvision
+ftfy

backend/python/diffusers/requirements-l4t12.txt

@@ -1,12 +0,0 @@
---extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu129/
-torch
-git+https://github.com/huggingface/diffusers
-transformers
-accelerate
-compel
-peft
-optimum-quanto
-numpy<2
-sentencepiece
-torchvision
-ftfy

backend/python/diffusers/requirements-l4t13.txt

@@ -1,13 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch
-git+https://github.com/huggingface/diffusers
-transformers
-accelerate
-compel
-peft
-optimum-quanto
-numpy<2
-sentencepiece
-torchvision
-ftfy
-chardet

backend/python/diffusers/requirements.txt

@@ -1,5 +1,5 @@
 setuptools
-grpcio==1.76.0
+grpcio==1.74.0
 pillow
 protobuf
 certifi

backend/python/diffusers/test.py

@@ -1,26 +1,15 @@
 """
-A test script to test the gRPC service and dynamic loader
+A test script to test the gRPC service
 """
 import unittest
 import subprocess
 import time
-from unittest.mock import patch, MagicMock
+import backend_pb2
+import backend_pb2_grpc
 
-# Import dynamic loader for testing (these don't need gRPC)
-import diffusers_dynamic_loader as loader
-from diffusers import DiffusionPipeline, StableDiffusionPipeline
-
-# Try to import gRPC modules - may not be available during unit testing
-try:
-    import grpc
-    import backend_pb2
-    import backend_pb2_grpc
-    GRPC_AVAILABLE = True
-except ImportError:
-    GRPC_AVAILABLE = False
+import grpc
 
 
-@unittest.skipUnless(GRPC_AVAILABLE, "gRPC modules not available")
 class TestBackendServicer(unittest.TestCase):
     """
     TestBackendServicer is the class that tests the gRPC service
@@ -42,7 +31,7 @@ class TestBackendServicer(unittest.TestCase):
         """
         This method tests if the server starts up successfully
         """
-        time.sleep(20)
+        time.sleep(10)
         try:
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
@@ -59,7 +48,7 @@ class TestBackendServicer(unittest.TestCase):
         """
         This method tests if the model is loaded successfully
         """
-        time.sleep(20)
+        time.sleep(10)
         try:
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
@@ -77,7 +66,7 @@ class TestBackendServicer(unittest.TestCase):
         """
         This method tests if the backend can generate images
         """
-        time.sleep(20)
+        time.sleep(10)
         try:
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
@@ -93,222 +82,3 @@ class TestBackendServicer(unittest.TestCase):
             self.fail("Image gen service failed")
         finally:
             self.tearDown()
-
-
-class TestDiffusersDynamicLoader(unittest.TestCase):
-    """Test cases for the diffusers dynamic loader functionality."""
-
-    @classmethod
-    def setUpClass(cls):
-        """Set up test fixtures - clear caches to ensure fresh discovery."""
-        # Reset the caches to ensure fresh discovery
-        loader._pipeline_registry = None
-        loader._task_aliases = None
-
-    def test_camel_to_kebab_conversion(self):
-        """Test CamelCase to kebab-case conversion."""
-        test_cases = [
-            ("StableDiffusionPipeline", "stable-diffusion-pipeline"),
-            ("StableDiffusionXLPipeline", "stable-diffusion-xl-pipeline"),
-            ("FluxPipeline", "flux-pipeline"),
-            ("DiffusionPipeline", "diffusion-pipeline"),
-        ]
-        for input_val, expected in test_cases:
-            with self.subTest(input=input_val):
-                result = loader._camel_to_kebab(input_val)
-                self.assertEqual(result, expected)
-
-    def test_extract_task_keywords(self):
-        """Test task keyword extraction from class names."""
-        # Test text-to-image detection
-        aliases = loader._extract_task_keywords("StableDiffusionPipeline")
-        self.assertIn("stable-diffusion", aliases)
-
-        # Test img2img detection
-        aliases = loader._extract_task_keywords("StableDiffusionImg2ImgPipeline")
-        self.assertIn("image-to-image", aliases)
-        self.assertIn("img2img", aliases)
-
-        # Test inpainting detection
-        aliases = loader._extract_task_keywords("StableDiffusionInpaintPipeline")
-        self.assertIn("inpainting", aliases)
-        self.assertIn("inpaint", aliases)
-
-        # Test depth2img detection
-        aliases = loader._extract_task_keywords("StableDiffusionDepth2ImgPipeline")
-        self.assertIn("depth-to-image", aliases)
-
-    def test_discover_pipelines_finds_known_classes(self):
-        """Test that pipeline discovery finds at least one known pipeline class."""
-        registry = loader.get_pipeline_registry()
-
-        # Check that the registry is not empty
-        self.assertGreater(len(registry), 0, "Pipeline registry should not be empty")
-
-        # Check for known pipeline classes
-        known_pipelines = [
-            "StableDiffusionPipeline",
-            "DiffusionPipeline",
-        ]
-
-        for pipeline_name in known_pipelines:
-            with self.subTest(pipeline=pipeline_name):
-                self.assertIn(
-                    pipeline_name,
-                    registry,
-                    f"Expected to find {pipeline_name} in registry"
-                )
-
-    def test_discover_pipelines_caches_results(self):
-        """Test that pipeline discovery results are cached."""
-        # Get registry twice
-        registry1 = loader.get_pipeline_registry()
-        registry2 = loader.get_pipeline_registry()
-
-        # Should be the same object (cached)
-        self.assertIs(registry1, registry2, "Registry should be cached")
-
-    def test_get_available_pipelines(self):
-        """Test getting list of available pipelines."""
-        available = loader.get_available_pipelines()
-
-        # Should return a list
-        self.assertIsInstance(available, list)
-
-        # Should contain known pipelines
-        self.assertIn("StableDiffusionPipeline", available)
-        self.assertIn("DiffusionPipeline", available)
-
-        # Should be sorted
-        self.assertEqual(available, sorted(available))
-
-    def test_get_available_tasks(self):
-        """Test getting list of available task aliases."""
-        tasks = loader.get_available_tasks()
-
-        # Should return a list
-        self.assertIsInstance(tasks, list)
-
-        # Should be sorted
-        self.assertEqual(tasks, sorted(tasks))
-
-    def test_resolve_pipeline_class_by_name(self):
-        """Test resolving pipeline class by exact name."""
-        cls = loader.resolve_pipeline_class(class_name="StableDiffusionPipeline")
-        self.assertEqual(cls, StableDiffusionPipeline)
-
-    def test_resolve_pipeline_class_by_name_case_insensitive(self):
-        """Test that class name resolution is case-insensitive."""
-        cls1 = loader.resolve_pipeline_class(class_name="StableDiffusionPipeline")
-        cls2 = loader.resolve_pipeline_class(class_name="stablediffusionpipeline")
-        self.assertEqual(cls1, cls2)
-
-    def test_resolve_pipeline_class_by_task(self):
-        """Test resolving pipeline class by task alias."""
-        # Get the registry to find available tasks
-        aliases = loader.get_task_aliases()
-
-        # Test with a common task that should be available
-        if "stable-diffusion" in aliases:
-            cls = loader.resolve_pipeline_class(task="stable-diffusion")
-            self.assertIsNotNone(cls)
-
-    def test_resolve_pipeline_class_unknown_name_raises(self):
-        """Test that resolving unknown class name raises ValueError with helpful message."""
-        with self.assertRaises(ValueError) as ctx:
-            loader.resolve_pipeline_class(class_name="NonExistentPipeline")
-
-        # Check that error message includes available pipelines
-        error_msg = str(ctx.exception)
-        self.assertIn("Unknown pipeline class", error_msg)
-        self.assertIn("Available pipelines", error_msg)
-
-    def test_resolve_pipeline_class_unknown_task_raises(self):
-        """Test that resolving unknown task raises ValueError with helpful message."""
-        with self.assertRaises(ValueError) as ctx:
-            loader.resolve_pipeline_class(task="nonexistent-task-xyz")
-
-        # Check that error message includes available tasks
-        error_msg = str(ctx.exception)
-        self.assertIn("Unknown task", error_msg)
-        self.assertIn("Available tasks", error_msg)
-
-    def test_resolve_pipeline_class_no_params_raises(self):
-        """Test that calling with no parameters raises helpful ValueError."""
-        with self.assertRaises(ValueError) as ctx:
-            loader.resolve_pipeline_class()
-
-        error_msg = str(ctx.exception)
-        self.assertIn("Must provide at least one of", error_msg)
-
-    def test_get_pipeline_info(self):
-        """Test getting pipeline information."""
-        info = loader.get_pipeline_info("StableDiffusionPipeline")
-
-        self.assertEqual(info['name'], "StableDiffusionPipeline")
-        self.assertIsInstance(info['aliases'], list)
-        self.assertIsInstance(info['supports_single_file'], bool)
-
-    def test_get_pipeline_info_unknown_raises(self):
-        """Test that getting info for unknown pipeline raises ValueError."""
-        with self.assertRaises(ValueError) as ctx:
-            loader.get_pipeline_info("NonExistentPipeline")
-
-        self.assertIn("Unknown pipeline", str(ctx.exception))
-
-    def test_discover_diffusers_classes_pipelines(self):
-        """Test generic class discovery for DiffusionPipeline."""
-        classes = loader.discover_diffusers_classes("DiffusionPipeline")
-
-        # Should return a dict
-        self.assertIsInstance(classes, dict)
-
-        # Should contain known pipeline classes
-        self.assertIn("DiffusionPipeline", classes)
-        self.assertIn("StableDiffusionPipeline", classes)
-
-    def test_discover_diffusers_classes_caches_results(self):
-        """Test that class discovery results are cached."""
-        classes1 = loader.discover_diffusers_classes("DiffusionPipeline")
-        classes2 = loader.discover_diffusers_classes("DiffusionPipeline")
-
-        # Should be the same object (cached)
-        self.assertIs(classes1, classes2)
-
-    def test_discover_diffusers_classes_exclude_base(self):
-        """Test discovering classes without base class."""
-        classes = loader.discover_diffusers_classes("DiffusionPipeline", include_base=False)
-
-        # Should still contain subclasses
-        self.assertIn("StableDiffusionPipeline", classes)
-
-    def test_get_available_classes(self):
-        """Test getting list of available classes for a base class."""
-        classes = loader.get_available_classes("DiffusionPipeline")
-
-        # Should return a sorted list
-        self.assertIsInstance(classes, list)
-        self.assertEqual(classes, sorted(classes))
-
-        # Should contain known classes
-        self.assertIn("StableDiffusionPipeline", classes)
-
-
-class TestDiffusersDynamicLoaderWithMocks(unittest.TestCase):
-    """Test cases using mocks to test edge cases."""
-
-    def test_load_pipeline_requires_model_id(self):
-        """Test that load_diffusers_pipeline requires model_id."""
-        with self.assertRaises(ValueError) as ctx:
-            loader.load_diffusers_pipeline(class_name="StableDiffusionPipeline")
-
-        self.assertIn("model_id is required", str(ctx.exception))
-
-    def test_resolve_with_model_id_uses_diffusion_pipeline_fallback(self):
-        """Test that resolving with only model_id falls back to DiffusionPipeline."""
-        # When model_id is provided, if hub lookup is not successful,
-        # should fall back to DiffusionPipeline.
-        # This tests the fallback behavior - the actual hub lookup may succeed
-        # or fail depending on network, but the fallback path should work.
-        cls = loader.resolve_pipeline_class(model_id="some/nonexistent/model")
-        self.assertEqual(cls, DiffusionPipeline)

backend/python/exllama2/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.76.0
+grpcio==1.74.0
 protobuf
 certifi
 wheel

backend/python/faster-whisper/requirements-cublas13.txt

@@ -1,9 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch==2.9.1
-faster-whisper
-opencv-python
-accelerate
-compel
-peft
-sentencepiece
-optimum-quanto

backend/python/kokoro/backend.py

@@ -64,15 +64,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Generate audio using Kokoro pipeline
             generator = self.pipeline(request.text, voice=voice)
             
-            speechs = []
-            # Get all the audio segment
+            # Get the first (and typically only) audio segment
             for i, (gs, ps, audio) in enumerate(generator):
-                speechs.append(audio)
+                # Save audio to the destination file
+                sf.write(request.dst, audio, 24000)
                 print(f"Generated audio segment {i}: gs={gs}, ps={ps}", file=sys.stderr)
-            # Merges the audio segments and writes them to the destination
-            speech = torch.cat(speechs, dim=0)
-            sf.write(request.dst, speech, 24000)
-
+                # For now, we only process the first segment
+                # If you need to handle multiple segments, you might want to modify this
+                break
+                
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
         

backend/python/kokoro/requirements-cublas13.txt

@@ -1,7 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch==2.9.1
-torchaudio==2.9.1
-transformers
-accelerate
-kokoro
-soundfile

backend/python/mlx/backend.py

@@ -14,13 +14,11 @@ import backend_pb2_grpc
 import grpc
 from mlx_lm import load, generate, stream_generate
 from mlx_lm.sample_utils import make_sampler
-from mlx_lm.models.cache import make_prompt_cache, can_trim_prompt_cache, trim_prompt_cache
+from mlx_lm.models.cache import make_prompt_cache
 import mlx.core as mx
 import base64
 import io
 
-from mlx_cache import ThreadSafeLRUPromptCache
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 # If MAX_WORKERS are specified in the environment use it, otherwise default to 1
@@ -120,16 +118,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 self.model, self.tokenizer = load(request.Model, tokenizer_config=tokenizer_config)
             else:
                 self.model, self.tokenizer = load(request.Model)
-
-            # Initialize thread-safe LRU prompt cache for efficient generation
-            max_cache_entries = self.options.get("max_cache_entries", 10)
-            self.max_kv_size = self.options.get("max_kv_size", None)
-            self.model_key = request.Model
-            self.lru_cache = ThreadSafeLRUPromptCache(
-                max_size=max_cache_entries,
-                can_trim_fn=can_trim_prompt_cache,
-                trim_fn=trim_prompt_cache,
-            )
+            
+            # Initialize prompt cache for efficient generation
+            max_kv_size = self.options.get("max_kv_size", None)
+            self.prompt_cache = make_prompt_cache(self.model, max_kv_size)
                 
         except Exception as err:
             print(f"Error loading MLX model {err=}, {type(err)=}", file=sys.stderr)
@@ -142,8 +134,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         """
         Generates text based on the given prompt and sampling parameters using MLX.
 
-        Uses thread-safe LRU prompt cache for efficient prefix reuse across requests.
-
         Args:
             request: The predict request.
             context: The gRPC context.
@@ -151,48 +141,31 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         Returns:
             backend_pb2.Reply: The predict result.
         """
-        prompt_cache = None
-        cache_key = None
-
         try:
-            # Prepare the prompt and tokenize for cache key
-            prompt_text = self._prepare_prompt(request)
-            cache_key = self._get_tokens_from_prompt(prompt_text)
-
-            # Fetch nearest cache (exact, shorter prefix, or create new)
-            prompt_cache, remaining_tokens = self.lru_cache.fetch_nearest_cache(
-                self.model_key, cache_key
-            )
-            if prompt_cache is None:
-                prompt_cache = make_prompt_cache(self.model, self.max_kv_size)
-                remaining_tokens = cache_key
-
+            # Prepare the prompt
+            prompt = self._prepare_prompt(request)
+            
             # Build generation parameters using request attributes and options
             max_tokens, sampler_params = self._build_generation_params(request)
-
-            print(f"Generating text with MLX - max_tokens: {max_tokens}, cache_hit: {len(remaining_tokens) < len(cache_key)}", file=sys.stderr)
-
+            
+            print(f"Generating text with MLX - max_tokens: {max_tokens}, sampler_params: {sampler_params}", file=sys.stderr)
+            
             # Create sampler with parameters
             sampler = make_sampler(**sampler_params)
-
-            # Use stream_generate to track generated tokens for cache key
-            generated_text = []
-            for response in stream_generate(
+            
+            # Generate text using MLX with proper parameters
+            response = generate(
                 self.model,
                 self.tokenizer,
-                prompt=remaining_tokens if remaining_tokens else cache_key,
+                prompt=prompt,
                 max_tokens=max_tokens,
                 sampler=sampler,
-                prompt_cache=prompt_cache,
-            ):
-                generated_text.append(response.text)
-                cache_key.append(response.token)
-
-            # Insert completed cache
-            self.lru_cache.insert_cache(self.model_key, cache_key, prompt_cache)
-
-            return backend_pb2.Reply(message=bytes(''.join(generated_text), encoding='utf-8'))
-
+                prompt_cache=self.prompt_cache,
+                verbose=False
+            )
+            
+            return backend_pb2.Reply(message=bytes(response, encoding='utf-8'))
+            
         except Exception as e:
             print(f"Error in MLX Predict: {e}", file=sys.stderr)
             context.set_code(grpc.StatusCode.INTERNAL)
@@ -221,8 +194,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         """
         Generates text based on the given prompt and sampling parameters, and streams the results using MLX.
 
-        Uses thread-safe LRU prompt cache for efficient prefix reuse across requests.
-
         Args:
             request: The predict stream request.
             context: The gRPC context.
@@ -230,56 +201,35 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         Yields:
             backend_pb2.Reply: Streaming predict results.
         """
-        prompt_cache = None
-        cache_key = None
-
         try:
-            # Prepare the prompt and tokenize for cache key
-            prompt_text = self._prepare_prompt(request)
-            cache_key = self._get_tokens_from_prompt(prompt_text)
-
-            # Fetch nearest cache (exact, shorter prefix, or create new)
-            prompt_cache, remaining_tokens = self.lru_cache.fetch_nearest_cache(
-                self.model_key, cache_key
-            )
-            if prompt_cache is None:
-                prompt_cache = make_prompt_cache(self.model, self.max_kv_size)
-                remaining_tokens = cache_key
-
+            # Prepare the prompt
+            prompt = self._prepare_prompt(request)
+            
             # Build generation parameters using request attributes and options
             max_tokens, sampler_params = self._build_generation_params(request, default_max_tokens=512)
-
-            print(f"Streaming text with MLX - max_tokens: {max_tokens}, cache_hit: {len(remaining_tokens) < len(cache_key)}", file=sys.stderr)
-
+            
+            print(f"Streaming text with MLX - max_tokens: {max_tokens}, sampler_params: {sampler_params}", file=sys.stderr)
+            
             # Create sampler with parameters
             sampler = make_sampler(**sampler_params)
-
+            
             # Stream text generation using MLX with proper parameters
             for response in stream_generate(
                 self.model,
                 self.tokenizer,
-                prompt=remaining_tokens if remaining_tokens else cache_key,
+                prompt=prompt,
                 max_tokens=max_tokens,
                 sampler=sampler,
-                prompt_cache=prompt_cache,
+                prompt_cache=self.prompt_cache,
             ):
-                cache_key.append(response.token)
                 yield backend_pb2.Reply(message=bytes(response.text, encoding='utf-8'))
-
+                
         except Exception as e:
             print(f"Error in MLX PredictStream: {e}", file=sys.stderr)
             context.set_code(grpc.StatusCode.INTERNAL)
             context.set_details(f"Streaming generation failed: {str(e)}")
             yield backend_pb2.Reply(message=bytes("", encoding='utf-8'))
 
-        finally:
-            # Always insert cache, even on interruption
-            if prompt_cache is not None and cache_key is not None:
-                try:
-                    self.lru_cache.insert_cache(self.model_key, cache_key, prompt_cache)
-                except Exception as e:
-                    print(f"Error inserting cache: {e}", file=sys.stderr)
-
     def _prepare_prompt(self, request):
         """
         Prepare the prompt for MLX generation, handling chat templates if needed.
@@ -296,31 +246,16 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             messages = []
             for msg in request.Messages:
                 messages.append({"role": msg.role, "content": msg.content})
-
+            
             prompt = self.tokenizer.apply_chat_template(
-                messages,
-                tokenize=False,
+                messages, 
+                tokenize=False, 
                 add_generation_prompt=True
             )
             return prompt
         else:
             return request.Prompt
 
-    def _get_tokens_from_prompt(self, prompt_text: str) -> List[int]:
-        """
-        Tokenize prompt text for cache key generation.
-
-        Args:
-            prompt_text: The prompt string to tokenize.
-
-        Returns:
-            List[int]: List of token IDs.
-        """
-        tokens = self.tokenizer.encode(prompt_text)
-        if hasattr(tokens, 'tolist'):
-            return tokens.tolist()
-        return list(tokens)
-
 
 
 
@@ -349,19 +284,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         top_p = getattr(request, 'TopP', 0.0)
         if top_p == 0.0:
             top_p = 1.0  # Default top_p
-
-        min_p = getattr(request, 'MinP', 0.0)
-        # min_p default of 0.0 means disabled (no filtering)
-
-        top_k = getattr(request, 'TopK', 0)
-        # top_k default of 0 means disabled (no filtering)
-
+        
         # Initialize sampler parameters
         sampler_params = {
             'temp': temp,
             'top_p': top_p,
-            'min_p': min_p,
-            'top_k': top_k,
             'xtc_threshold': 0.0,
             'xtc_probability': 0.0,
         }
@@ -381,9 +308,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             sampler_option_mapping = {
                 'temp': 'temp',
                 'temperature': 'temp',  # alias
-                'top_p': 'top_p',
-                'min_p': 'min_p',
-                'top_k': 'top_k',
+                'top_p': 'top_p', 
                 'xtc_threshold': 'xtc_threshold',
                 'xtc_probability': 'xtc_probability',
             }

backend/python/mlx/mlx_cache.py

@@ -1,266 +0,0 @@
-"""
-Thread-safe LRU prompt cache for MLX-based backends.
-
-Ported from mlx_lm/server.py (MIT License, Copyright 2023-2024 Apple Inc.)
-with thread-safety additions for LocalAI's gRPC backend.
-
-Usage:
-    from mlx_cache import ThreadSafeLRUPromptCache
-
-    # In LoadModel:
-    self.lru_cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    # In Predict/PredictStream:
-    prompt_cache, remaining_tokens = self.lru_cache.fetch_nearest_cache(model_key, tokens)
-    # ... generate ...
-    self.lru_cache.insert_cache(model_key, tokens, prompt_cache)
-"""
-import copy
-import threading
-from collections import deque
-from dataclasses import dataclass
-from typing import Any, List, Optional, Tuple
-
-
-@dataclass
-class CacheEntry:
-    """A cache entry with reference counting."""
-    prompt_cache: List[Any]
-    count: int
-
-
-@dataclass
-class SearchResult:
-    """Result of searching the cache trie."""
-    model: Any
-    exact: Optional[List[int]]
-    shorter: Optional[List[int]]
-    longer: Optional[List[int]]
-    common_prefix: int
-
-
-class ThreadSafeLRUPromptCache:
-    """
-    Thread-safe LRU cache with prefix matching for prompt KV caches.
-
-    This cache stores KV caches keyed by token sequences and supports:
-    - Exact match: Return the cache for the exact token sequence
-    - Shorter prefix match: Return a cache for a prefix of the tokens
-    - Longer prefix match: If a longer sequence is cached and can be trimmed
-    - LRU eviction: When max_size is exceeded, evict least recently used
-
-    Thread safety is provided via a threading.Lock that protects all
-    cache operations.
-
-    Args:
-        max_size: Maximum number of cache entries (default: 10)
-        can_trim_fn: Optional function to check if a cache can be trimmed
-        trim_fn: Optional function to trim a cache
-    """
-
-    def __init__(
-        self,
-        max_size: int = 10,
-        can_trim_fn: Optional[Any] = None,
-        trim_fn: Optional[Any] = None,
-    ):
-        self.max_size = max_size
-        self._cache = {}
-        self._lru = deque()
-        self._lock = threading.Lock()
-
-        # Optional trim functions (for longer prefix reuse)
-        self._can_trim_fn = can_trim_fn
-        self._trim_fn = trim_fn
-
-    def _search(self, model, tokens: List[int]) -> SearchResult:
-        """
-        Search the cache for a prompt cache. Return exact or close match.
-
-        The cache is organized as a trie where each node is keyed by a token.
-        This allows efficient prefix matching.
-        """
-        if model not in self._cache:
-            return SearchResult(model, None, None, None, 0)
-
-        current = self._cache[model]
-        last_cache_index = -1
-        index = 0
-
-        # Traverse the trie following the token sequence
-        while index < len(tokens) and tokens[index] in current:
-            current = current[tokens[index]]
-            if "cache" in current:
-                last_cache_index = index
-            index += 1
-
-        # Exact match - no need to search for longer or shorter caches
-        if last_cache_index == len(tokens) - 1:
-            return SearchResult(model, tuple(tokens), None, None, 0)
-
-        # Find the shorter cache (a prefix that has a cache)
-        # Note: Uses > 0 (not >= 0) to match upstream mlx_lm/server.py behavior.
-        # Single-token prefixes are not matched, which allows longer cached
-        # sequences to be preferred for trimming. This is acceptable because
-        # real prompts with chat templates are always many tokens.
-        shorter = None
-        if last_cache_index > 0:
-            shorter = tuple(tokens[: last_cache_index + 1])
-
-        # Check for caches that are longer than our token sequence
-        longer = None
-        common_prefix = index
-        if index > 0 and last_cache_index <= 0:
-            best = None
-            stack = [(current, [])]
-            while stack:
-                current, extra = stack.pop()
-                if "cache" in current:
-                    if best is None or len(extra) < len(best):
-                        best = extra
-                else:
-                    for tok in current:
-                        stack.append((current[tok], extra + [tok]))
-            if best is not None:
-                longer = tuple(tokens[:index] + best)
-
-        return SearchResult(model, None, shorter, longer, common_prefix)
-
-    def _get(self, model, tokens: Tuple[int, ...]) -> CacheEntry:
-        """Get a cache entry by traversing the trie."""
-        current = self._cache[model]
-        for tok in tokens:
-            current = current[tok]
-        return current["cache"]
-
-    def _delete(self, model, tokens: Tuple[int, ...]) -> None:
-        """Delete a cache entry and clean up empty trie nodes."""
-        path = [self._cache[model]]
-        for tok in tokens:
-            path.append(path[-1][tok])
-        del path[-1]["cache"]
-
-        # Clean up empty nodes bottom-up
-        for i in reversed(range(len(tokens))):
-            d_prev, d, t = path[i], path[i + 1], tokens[i]
-            if len(d) > 0:
-                break
-            del d_prev[t]
-
-    def _extract(self, model, tokens: Tuple[int, ...]) -> CacheEntry:
-        """
-        Extract a cache entry for exclusive use.
-
-        If the entry has count > 1, deep copy and decrement.
-        If count == 1, remove from cache entirely.
-        """
-        cache_entry = self._get(model, tokens)
-        if cache_entry.count == 1:
-            self._delete(model, tokens)
-            self._lru.remove((model, tokens))
-            return cache_entry
-
-        cache_entry.count -= 1
-        return CacheEntry(
-            copy.deepcopy(cache_entry.prompt_cache),
-            1,
-        )
-
-    def fetch_nearest_cache(
-        self, model, tokens: List[int]
-    ) -> Tuple[Optional[List[Any]], List[int]]:
-        """
-        Fetch the nearest cache for the given token sequence.
-
-        Thread-safe. Returns (cache, remaining_tokens) where:
-        - cache: The KV cache to use (or None if no cache found)
-        - remaining_tokens: Tokens that still need to be processed
-
-        Args:
-            model: Model identifier (used to namespace caches)
-            tokens: The full token sequence for the prompt
-
-        Returns:
-            Tuple of (prompt_cache, remaining_tokens)
-        """
-        with self._lock:
-            tokens_tuple = tuple(tokens)
-            result = self._search(model, tokens)
-
-            # Exact match - extract and return
-            if result.exact is not None:
-                cache_entry = self._extract(result.model, result.exact)
-                return cache_entry.prompt_cache, []
-
-            # Shorter prefix match - extract and return remaining
-            if result.shorter is not None:
-                cache_entry = self._extract(result.model, result.shorter)
-                prefix_len = len(result.shorter)
-                return cache_entry.prompt_cache, list(tokens[prefix_len:])
-
-            # Longer prefix match - try to trim if possible
-            if result.longer is not None and self._can_trim_fn is not None:
-                cache_entry = self._get(result.model, result.longer)
-                if self._can_trim_fn(cache_entry.prompt_cache):
-                    # Deep copy and trim
-                    trimmed_cache = copy.deepcopy(cache_entry.prompt_cache)
-                    prefix = min(len(tokens) - 1, result.common_prefix)
-                    num_to_trim = len(result.longer) - prefix
-                    if self._trim_fn is not None:
-                        self._trim_fn(trimmed_cache, num_to_trim)
-                    return trimmed_cache, list(tokens[prefix:])
-
-            # No match found
-            return None, list(tokens)
-
-    def insert_cache(
-        self, model, tokens: List[int], prompt_cache: List[Any]
-    ) -> None:
-        """
-        Insert a cache entry after generation completes.
-
-        Thread-safe. Handles LRU eviction if max_size is exceeded.
-
-        Args:
-            model: Model identifier (used to namespace caches)
-            tokens: The full token sequence (prompt + generated)
-            prompt_cache: The KV cache to store
-        """
-        with self._lock:
-            tokens_tuple = tuple(tokens)
-
-            if model not in self._cache:
-                self._cache[model] = {}
-            current = self._cache[model]
-
-            # Build trie path
-            for tok in tokens_tuple:
-                if tok not in current:
-                    current[tok] = {}
-                current = current[tok]
-
-            # Update or create entry
-            if "cache" in current:
-                current["cache"].count += 1
-                self._lru.remove((model, tokens_tuple))
-            else:
-                current["cache"] = CacheEntry(prompt_cache, 1)
-
-            # Update LRU order
-            self._lru.append((model, tokens_tuple))
-
-            # Evict if over capacity
-            if len(self._lru) > self.max_size:
-                evict_model, evict_tokens = self._lru.popleft()
-                self._delete(evict_model, evict_tokens)
-
-    def clear(self) -> None:
-        """Clear all cache entries. Thread-safe."""
-        with self._lock:
-            self._cache.clear()
-            self._lru.clear()
-
-    def __len__(self) -> int:
-        """Return the number of cache entries. Thread-safe."""
-        with self._lock:
-            return len(self._lru)

backend/python/mlx/test.py

@@ -1,11 +1,18 @@
 import unittest
 import subprocess
 import time
-
-import grpc
 import backend_pb2
 import backend_pb2_grpc
 
+import grpc
+
+import unittest
+import subprocess
+import time
+import grpc
+import backend_pb2_grpc
+import backend_pb2
+
 class TestBackendServicer(unittest.TestCase):
     """
     TestBackendServicer is the class that tests the gRPC service.
@@ -40,9 +47,9 @@ class TestBackendServicer(unittest.TestCase):
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
                 stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit"))
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
                 self.assertTrue(response.success)
-                self.assertEqual(response.message, "MLX model loaded successfully")
+                self.assertEqual(response.message, "Model loaded successfully")
         except Exception as err:
             print(err)
             self.fail("LoadModel service failed")
@@ -57,7 +64,7 @@ class TestBackendServicer(unittest.TestCase):
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
                 stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit"))
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
                 self.assertTrue(response.success)
                 req = backend_pb2.PredictOptions(Prompt="The capital of France is")
                 resp = stub.Predict(req)
@@ -77,7 +84,7 @@ class TestBackendServicer(unittest.TestCase):
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
                 stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit"))
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
                 self.assertTrue(response.success)
 
                 req = backend_pb2.PredictOptions(
@@ -88,13 +95,26 @@ class TestBackendServicer(unittest.TestCase):
                     TopK=40,
                     PresencePenalty=0.1,
                     FrequencyPenalty=0.2,
+                    RepetitionPenalty=1.1,
                     MinP=0.05,
                     Seed=42,
                     StopPrompts=["\n"],
+                    StopTokenIds=[50256],
+                    BadWords=["badword"],
+                    IncludeStopStrInOutput=True,
                     IgnoreEOS=True,
+                    MinTokens=5,
+                    Logprobs=5,
+                    PromptLogprobs=5,
+                    SkipSpecialTokens=True,
+                    SpacesBetweenSpecialTokens=True,
+                    TruncatePromptTokens=10,
+                    GuidedDecoding=True,
+                    N=2,
                 )
                 resp = stub.Predict(req)
                 self.assertIsNotNone(resp.message)
+                self.assertIsNotNone(resp.logprobs)
         except Exception as err:
             print(err)
             self.fail("sampling params service failed")
@@ -123,112 +143,4 @@ class TestBackendServicer(unittest.TestCase):
             print(err)
             self.fail("Embedding service failed")
         finally:
-            self.tearDown()
-
-    def test_concurrent_requests(self):
-        """
-        This method tests that concurrent requests don't corrupt each other's cache state.
-        This is a regression test for the race condition in the original implementation.
-        """
-        import concurrent.futures
-
-        try:
-            self.setUp()
-            with grpc.insecure_channel("localhost:50051") as channel:
-                stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit"))
-                self.assertTrue(response.success)
-
-                def make_request(prompt):
-                    req = backend_pb2.PredictOptions(Prompt=prompt, Tokens=20)
-                    return stub.Predict(req)
-
-                # Run 5 concurrent requests with different prompts
-                prompts = [
-                    "The capital of France is",
-                    "The capital of Germany is",
-                    "The capital of Italy is",
-                    "The capital of Spain is",
-                    "The capital of Portugal is",
-                ]
-
-                with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
-                    futures = [executor.submit(make_request, p) for p in prompts]
-                    results = [f.result() for f in concurrent.futures.as_completed(futures)]
-
-                # All results should be non-empty
-                messages = [r.message for r in results]
-                self.assertTrue(all(len(m) > 0 for m in messages), "All requests should return non-empty responses")
-                print(f"Concurrent test passed: {len(messages)} responses received")
-
-        except Exception as err:
-            print(err)
-            self.fail("Concurrent requests test failed")
-        finally:
-            self.tearDown()
-
-    def test_cache_reuse(self):
-        """
-        This method tests that repeated prompts reuse cached KV states.
-        The second request should benefit from the cached prompt processing.
-        """
-        try:
-            self.setUp()
-            with grpc.insecure_channel("localhost:50051") as channel:
-                stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit"))
-                self.assertTrue(response.success)
-
-                prompt = "The quick brown fox jumps over the lazy dog. "
-
-                # First request - populates cache
-                req1 = backend_pb2.PredictOptions(Prompt=prompt, Tokens=10)
-                resp1 = stub.Predict(req1)
-                self.assertIsNotNone(resp1.message)
-
-                # Second request with same prompt - should reuse cache
-                req2 = backend_pb2.PredictOptions(Prompt=prompt, Tokens=10)
-                resp2 = stub.Predict(req2)
-                self.assertIsNotNone(resp2.message)
-
-                print(f"Cache reuse test passed: first={len(resp1.message)} bytes, second={len(resp2.message)} bytes")
-
-        except Exception as err:
-            print(err)
-            self.fail("Cache reuse test failed")
-        finally:
-            self.tearDown()
-
-    def test_prefix_cache_reuse(self):
-        """
-        This method tests that prompts sharing a common prefix benefit from cached KV states.
-        """
-        try:
-            self.setUp()
-            with grpc.insecure_channel("localhost:50051") as channel:
-                stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit"))
-                self.assertTrue(response.success)
-
-                # First request with base prompt
-                prompt_base = "Once upon a time in a land far away, "
-                req1 = backend_pb2.PredictOptions(Prompt=prompt_base, Tokens=10)
-                resp1 = stub.Predict(req1)
-                self.assertIsNotNone(resp1.message)
-
-                # Second request with extended prompt (same prefix)
-                prompt_extended = prompt_base + "there lived a brave knight who "
-                req2 = backend_pb2.PredictOptions(Prompt=prompt_extended, Tokens=10)
-                resp2 = stub.Predict(req2)
-                self.assertIsNotNone(resp2.message)
-
-                print(f"Prefix cache test passed: base={len(resp1.message)} bytes, extended={len(resp2.message)} bytes")
-
-        except Exception as err:
-            print(err)
-            self.fail("Prefix cache reuse test failed")
-        finally:
-            self.tearDown()
-
-
-# Unit tests for ThreadSafeLRUPromptCache are in test_mlx_cache.py
+            self.tearDown()

backend/python/mlx/test_mlx_cache.py

@@ -1,480 +0,0 @@
-"""
-Comprehensive unit tests for ThreadSafeLRUPromptCache.
-
-Tests all cache operation modes:
-- Exact match
-- Shorter prefix match
-- Longer prefix match (with trimming)
-- No match
-- LRU eviction
-- Reference counting
-- Multi-model namespacing
-- Thread safety with data integrity verification
-"""
-import unittest
-import concurrent.futures
-import threading
-import copy
-from mlx_cache import ThreadSafeLRUPromptCache
-
-
-class TestCacheExactMatch(unittest.TestCase):
-    """Tests for exact match cache behavior."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    def test_exact_match_returns_cache_and_empty_remaining(self):
-        """Exact match should return the cache with no remaining tokens."""
-        tokens = [1, 2, 3, 4, 5]
-        mock_cache = ["kv_cache_data"]
-
-        self.cache.insert_cache("model1", tokens, mock_cache)
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", tokens)
-
-        self.assertEqual(result_cache, mock_cache)
-        self.assertEqual(remaining, [])
-
-    def test_exact_match_extracts_and_removes_from_cache(self):
-        """Fetching exact match with count=1 should remove entry from cache."""
-        tokens = [1, 2, 3]
-        self.cache.insert_cache("model1", tokens, ["cache"])
-
-        self.assertEqual(len(self.cache), 1)
-
-        # First fetch extracts the entry
-        self.cache.fetch_nearest_cache("model1", tokens)
-
-        # Cache should now be empty
-        self.assertEqual(len(self.cache), 0)
-
-        # Second fetch should return None (no match)
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", tokens)
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, tokens)
-
-
-class TestCacheShorterPrefix(unittest.TestCase):
-    """Tests for shorter prefix match behavior."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    def test_shorter_prefix_returns_cache_with_remaining_tokens(self):
-        """When cached prefix is shorter, return cache and remaining suffix."""
-        short_tokens = [1, 2, 3]
-        long_tokens = [1, 2, 3, 4, 5, 6]
-        mock_cache = ["prefix_cache"]
-
-        self.cache.insert_cache("model1", short_tokens, mock_cache)
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", long_tokens)
-
-        self.assertEqual(result_cache, mock_cache)
-        self.assertEqual(remaining, [4, 5, 6])
-
-    def test_shorter_prefix_correct_remaining_calculation(self):
-        """Verify remaining tokens are calculated correctly for various prefix lengths."""
-        # Note: Single-token prefixes ([1] -> [1,2,3]) are deliberately not matched
-        # to allow longer cached sequences to be preferred for trimming.
-        # This matches upstream mlx_lm/server.py behavior.
-        test_cases = [
-            # (cached_tokens, requested_tokens, expected_remaining)
-            ([1, 2], [1, 2, 3, 4, 5], [3, 4, 5]),
-            ([10, 20, 30, 40], [10, 20, 30, 40, 50], [50]),
-        ]
-
-        for cached, requested, expected_remaining in test_cases:
-            with self.subTest(cached=cached, requested=requested):
-                cache = ThreadSafeLRUPromptCache(max_size=10)
-                cache.insert_cache("model", cached, ["cache"])
-                result_cache, remaining = cache.fetch_nearest_cache("model", requested)
-
-                self.assertIsNotNone(result_cache)
-                self.assertEqual(remaining, expected_remaining)
-
-    def test_single_token_prefix_not_matched(self):
-        """Single-token prefixes are not matched (by design, matches upstream).
-
-        This allows longer cached sequences to be preferred for trimming,
-        which provides better KV cache reuse. Single-token caches are rare
-        in practice since real prompts with chat templates are many tokens.
-        """
-        cache = ThreadSafeLRUPromptCache(max_size=10)
-        cache.insert_cache("model", [1], ["cache"])
-
-        result_cache, remaining = cache.fetch_nearest_cache("model", [1, 2, 3])
-
-        # Single-token prefix is NOT matched
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, [1, 2, 3])
-
-
-class TestCacheLongerPrefix(unittest.TestCase):
-    """Tests for longer prefix match behavior (trimming)."""
-
-    def setUp(self):
-        # Track trim calls for verification
-        self.trim_calls = []
-
-        def mock_can_trim(cache):
-            return True
-
-        def mock_trim(cache, num_to_trim):
-            self.trim_calls.append(num_to_trim)
-            # Simulate trimming by modifying the cache
-            cache.append(f"trimmed_{num_to_trim}")
-
-        self.cache = ThreadSafeLRUPromptCache(
-            max_size=10,
-            can_trim_fn=mock_can_trim,
-            trim_fn=mock_trim,
-        )
-
-    def test_longer_prefix_triggers_trim(self):
-        """When cached sequence is longer, should trim to match requested prefix."""
-        long_tokens = [1, 2, 3, 4, 5]
-        short_tokens = [1, 2, 3]
-
-        self.cache.insert_cache("model1", long_tokens, ["original_cache"])
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", short_tokens)
-
-        # Should have called trim
-        self.assertTrue(len(self.trim_calls) > 0, "trim_fn should have been called")
-        # Result should be a trimmed copy, not the original
-        self.assertIn("trimmed_", str(result_cache))
-
-    def test_longer_prefix_without_trim_fn_returns_no_match(self):
-        """Without trim functions, longer prefix should not match."""
-        cache_no_trim = ThreadSafeLRUPromptCache(max_size=10)
-
-        long_tokens = [1, 2, 3, 4, 5]
-        short_tokens = [1, 2, 3]
-
-        cache_no_trim.insert_cache("model1", long_tokens, ["cache"])
-        result_cache, remaining = cache_no_trim.fetch_nearest_cache("model1", short_tokens)
-
-        # Without trim_fn, should return no match
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, short_tokens)
-
-    def test_longer_prefix_can_trim_false_returns_no_match(self):
-        """When can_trim_fn returns False, should not attempt trim."""
-        cache = ThreadSafeLRUPromptCache(
-            max_size=10,
-            can_trim_fn=lambda c: False,
-            trim_fn=lambda c, n: None,
-        )
-
-        cache.insert_cache("model1", [1, 2, 3, 4, 5], ["cache"])
-        result_cache, remaining = cache.fetch_nearest_cache("model1", [1, 2, 3])
-
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, [1, 2, 3])
-
-
-class TestCacheNoMatch(unittest.TestCase):
-    """Tests for no match behavior."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    def test_empty_cache_returns_none(self):
-        """Empty cache should return None and all tokens as remaining."""
-        tokens = [1, 2, 3]
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", tokens)
-
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, tokens)
-
-    def test_different_prefix_returns_none(self):
-        """Tokens with different prefix should not match."""
-        self.cache.insert_cache("model1", [1, 2, 3], ["cache"])
-
-        # Completely different tokens
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", [4, 5, 6])
-
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, [4, 5, 6])
-
-    def test_partial_prefix_mismatch_returns_none(self):
-        """Tokens that diverge mid-sequence should not match."""
-        self.cache.insert_cache("model1", [1, 2, 3], ["cache"])
-
-        # Same start but diverges
-        result_cache, remaining = self.cache.fetch_nearest_cache("model1", [1, 2, 99])
-
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, [1, 2, 99])
-
-    def test_wrong_model_returns_none(self):
-        """Different model key should not match."""
-        self.cache.insert_cache("model1", [1, 2, 3], ["cache"])
-
-        result_cache, remaining = self.cache.fetch_nearest_cache("model2", [1, 2, 3])
-
-        self.assertIsNone(result_cache)
-        self.assertEqual(remaining, [1, 2, 3])
-
-
-class TestCacheLRUEviction(unittest.TestCase):
-    """Tests for LRU eviction behavior."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=3)
-
-    def test_evicts_oldest_when_full(self):
-        """Should evict least recently used entry when capacity exceeded."""
-        self.cache.insert_cache("model", [1], ["cache1"])
-        self.cache.insert_cache("model", [2], ["cache2"])
-        self.cache.insert_cache("model", [3], ["cache3"])
-
-        self.assertEqual(len(self.cache), 3)
-
-        # Insert 4th entry - should evict [1]
-        self.cache.insert_cache("model", [4], ["cache4"])
-
-        self.assertEqual(len(self.cache), 3)
-
-        # [1] should be evicted
-        result, _ = self.cache.fetch_nearest_cache("model", [1])
-        self.assertIsNone(result)
-
-        # [2], [3], [4] should still exist
-        for tokens in [[2], [3], [4]]:
-            # Re-insert since fetch extracts
-            self.cache.insert_cache("model", tokens, [f"cache{tokens[0]}"])
-
-        result2, _ = self.cache.fetch_nearest_cache("model", [2])
-        self.assertIsNotNone(result2)
-
-    def test_access_updates_lru_order(self):
-        """Accessing an entry should move it to most recently used."""
-        self.cache.insert_cache("model", [1], ["cache1"])
-        self.cache.insert_cache("model", [2], ["cache2"])
-        self.cache.insert_cache("model", [3], ["cache3"])
-
-        # Access [1] to make it most recently used
-        cache1, _ = self.cache.fetch_nearest_cache("model", [1])
-        # Re-insert it (simulating normal usage pattern)
-        self.cache.insert_cache("model", [1], cache1)
-
-        # Now insert two more entries - should evict [2] then [3], not [1]
-        self.cache.insert_cache("model", [4], ["cache4"])
-        self.cache.insert_cache("model", [5], ["cache5"])
-
-        # [1] should still exist (was accessed, so not evicted)
-        result1, _ = self.cache.fetch_nearest_cache("model", [1])
-        self.assertIsNotNone(result1)
-
-        # [2] should be evicted (was oldest after [1] was accessed)
-        result2, _ = self.cache.fetch_nearest_cache("model", [2])
-        self.assertIsNone(result2)
-
-
-class TestCacheReferenceCount(unittest.TestCase):
-    """Tests for reference counting behavior."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    def test_multiple_inserts_increment_count(self):
-        """Inserting same tokens multiple times should increment count."""
-        tokens = [1, 2, 3]
-
-        self.cache.insert_cache("model", tokens, ["cache"])
-        self.cache.insert_cache("model", tokens, ["cache"])
-        self.cache.insert_cache("model", tokens, ["cache"])
-
-        # Should still be one entry (with count=3 internally)
-        self.assertEqual(len(self.cache), 1)
-
-        # First two fetches should return copies (count decremented)
-        result1, _ = self.cache.fetch_nearest_cache("model", tokens)
-        self.assertIsNotNone(result1)
-
-        result2, _ = self.cache.fetch_nearest_cache("model", tokens)
-        self.assertIsNotNone(result2)
-
-        # Third fetch extracts the last reference
-        result3, _ = self.cache.fetch_nearest_cache("model", tokens)
-        self.assertIsNotNone(result3)
-
-        # Fourth fetch should return None (entry fully extracted)
-        result4, _ = self.cache.fetch_nearest_cache("model", tokens)
-        self.assertIsNone(result4)
-
-    def test_extract_with_high_count_returns_deep_copy(self):
-        """When count > 1, extract should return a deep copy."""
-        tokens = [1, 2, 3]
-        original_cache = [{"nested": "data"}]
-
-        self.cache.insert_cache("model", tokens, original_cache)
-        self.cache.insert_cache("model", tokens, original_cache)  # count=2
-
-        result1, _ = self.cache.fetch_nearest_cache("model", tokens)
-
-        # Modify the returned cache
-        result1[0]["nested"] = "modified"
-
-        # Second fetch should get unmodified copy
-        result2, _ = self.cache.fetch_nearest_cache("model", tokens)
-        self.assertEqual(result2[0]["nested"], "data")
-
-
-class TestCacheMultiModel(unittest.TestCase):
-    """Tests for multi-model namespacing."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    def test_same_tokens_different_models_are_separate(self):
-        """Same token sequence under different models should be independent."""
-        tokens = [1, 2, 3]
-
-        self.cache.insert_cache("model_a", tokens, ["cache_a"])
-        self.cache.insert_cache("model_b", tokens, ["cache_b"])
-
-        self.assertEqual(len(self.cache), 2)
-
-        result_a, _ = self.cache.fetch_nearest_cache("model_a", tokens)
-        result_b, _ = self.cache.fetch_nearest_cache("model_b", tokens)
-
-        self.assertEqual(result_a, ["cache_a"])
-        self.assertEqual(result_b, ["cache_b"])
-
-    def test_eviction_across_models(self):
-        """LRU eviction should work across different models."""
-        cache = ThreadSafeLRUPromptCache(max_size=3)
-
-        cache.insert_cache("model_a", [1], ["a1"])
-        cache.insert_cache("model_b", [1], ["b1"])
-        cache.insert_cache("model_a", [2], ["a2"])
-
-        self.assertEqual(len(cache), 3)
-
-        # Insert 4th - should evict model_a:[1] (oldest)
-        cache.insert_cache("model_b", [2], ["b2"])
-
-        result, _ = cache.fetch_nearest_cache("model_a", [1])
-        self.assertIsNone(result)
-
-
-class TestCacheThreadSafety(unittest.TestCase):
-    """Tests for thread safety with data integrity verification."""
-
-    def test_concurrent_inserts_no_data_loss(self):
-        """Concurrent inserts should not lose data."""
-        cache = ThreadSafeLRUPromptCache(max_size=100)
-        num_threads = 10
-        inserts_per_thread = 20
-
-        def insert_entries(thread_id):
-            for i in range(inserts_per_thread):
-                tokens = [thread_id, i]
-                cache.insert_cache("model", tokens, [f"cache_{thread_id}_{i}"])
-
-        with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
-            futures = [executor.submit(insert_entries, tid) for tid in range(num_threads)]
-            concurrent.futures.wait(futures)
-
-        # Verify expected number of entries (may be less due to LRU eviction with max_size=100)
-        # But should be exactly 100 since we inserted exactly 200 and max_size is 100
-        self.assertEqual(len(cache), 100)
-
-    def test_concurrent_fetch_and_insert_no_corruption(self):
-        """Concurrent fetches and inserts should not corrupt data."""
-        cache = ThreadSafeLRUPromptCache(max_size=50)
-        errors = []
-        lock = threading.Lock()
-
-        # Pre-populate with known data
-        for i in range(20):
-            cache.insert_cache("model", [i], [f"original_{i}"])
-
-        def fetch_and_verify(thread_id):
-            try:
-                for _ in range(50):
-                    token_id = thread_id % 20
-                    result, remaining = cache.fetch_nearest_cache("model", [token_id])
-
-                    if result is not None:
-                        # Verify data integrity
-                        expected_prefix = f"original_{token_id}"
-                        if not str(result[0]).startswith("original_"):
-                            with lock:
-                                errors.append(f"Corrupted data: {result}")
-
-                        # Re-insert to keep cache populated
-                        cache.insert_cache("model", [token_id], result)
-
-            except Exception as e:
-                with lock:
-                    errors.append(str(e))
-
-        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-            futures = [executor.submit(fetch_and_verify, tid) for tid in range(10)]
-            concurrent.futures.wait(futures)
-
-        self.assertEqual(errors, [], f"Thread safety errors: {errors}")
-
-    def test_concurrent_operations_maintain_cache_bounds(self):
-        """Cache size should never exceed max_size under concurrent operations."""
-        max_size = 10
-        cache = ThreadSafeLRUPromptCache(max_size=max_size)
-        size_violations = []
-        lock = threading.Lock()
-
-        def random_operations(thread_id):
-            import random
-            for i in range(100):
-                tokens = [random.randint(0, 50)]
-                if random.random() < 0.7:
-                    cache.insert_cache("model", tokens, [f"cache_{thread_id}_{i}"])
-                else:
-                    cache.fetch_nearest_cache("model", tokens)
-
-                current_size = len(cache)
-                if current_size > max_size:
-                    with lock:
-                        size_violations.append(current_size)
-
-        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-            futures = [executor.submit(random_operations, tid) for tid in range(10)]
-            concurrent.futures.wait(futures)
-
-        self.assertEqual(size_violations, [], f"Size exceeded max: {size_violations}")
-        self.assertLessEqual(len(cache), max_size)
-
-
-class TestCacheClear(unittest.TestCase):
-    """Tests for cache clear operation."""
-
-    def setUp(self):
-        self.cache = ThreadSafeLRUPromptCache(max_size=10)
-
-    def test_clear_removes_all_entries(self):
-        """Clear should remove all entries."""
-        self.cache.insert_cache("model1", [1, 2], ["cache1"])
-        self.cache.insert_cache("model2", [3, 4], ["cache2"])
-        self.cache.insert_cache("model1", [5, 6], ["cache3"])
-
-        self.assertEqual(len(self.cache), 3)
-
-        self.cache.clear()
-
-        self.assertEqual(len(self.cache), 0)
-
-    def test_clear_allows_new_inserts(self):
-        """After clear, new inserts should work normally."""
-        self.cache.insert_cache("model", [1], ["cache1"])
-        self.cache.clear()
-        self.cache.insert_cache("model", [2], ["cache2"])
-
-        self.assertEqual(len(self.cache), 1)
-
-        result, _ = self.cache.fetch_nearest_cache("model", [2])
-        self.assertEqual(result, ["cache2"])
-
-
-if __name__ == "__main__":
-    unittest.main()

backend/python/neutts/Makefile

@@ -1,23 +0,0 @@
-.PHONY: neutts
-neutts:
-	bash install.sh
-
-.PHONY: run
-run: neutts
-	@echo "Running neutts..."
-	bash run.sh
-	@echo "neutts run."
-
-.PHONY: test
-test: neutts
-	@echo "Testing neutts..."
-	bash test.sh
-	@echo "neutts tested."
-
-.PHONY: protogen-clean
-protogen-clean:
-	$(RM) backend_pb2_grpc.py backend_pb2.py
-
-.PHONY: clean
-clean: protogen-clean
-	rm -rf venv __pycache__

backend/python/neutts/backend.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-"""
-This is an extra gRPC server of LocalAI for NeuTTSAir
-"""
-from concurrent import futures
-import time
-import argparse
-import signal
-import sys
-import os
-import backend_pb2
-import backend_pb2_grpc
-import torch
-from neuttsair.neutts import NeuTTSAir
-import soundfile as sf
-
-import grpc
-
-def is_float(s):
-    """Check if a string can be converted to float."""
-    try:
-        float(s)
-        return True
-    except ValueError:
-        return False
-def is_int(s):
-    """Check if a string can be converted to int."""
-    try:
-        int(s)
-        return True
-    except ValueError:
-        return False
-
-_ONE_DAY_IN_SECONDS = 60 * 60 * 24
-
-# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
-MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
-
-# Implement the BackendServicer class with the service methods
-class BackendServicer(backend_pb2_grpc.BackendServicer):
-    """
-    BackendServicer is the class that implements the gRPC service
-    """
-    def Health(self, request, context):
-        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
-    def LoadModel(self, request, context):
-
-        # Get device
-        # device = "cuda" if request.CUDA else "cpu"
-        if torch.cuda.is_available():
-            print("CUDA is available", file=sys.stderr)
-            device = "cuda"
-        else:
-            print("CUDA is not available", file=sys.stderr)
-            device = "cpu"
-        mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
-        if mps_available:
-            device = "mps"
-        if not torch.cuda.is_available() and request.CUDA:
-            return backend_pb2.Result(success=False, message="CUDA is not available")
-
-
-        options = request.Options
-
-        # empty dict
-        self.options = {}
-        self.ref_text = None
-
-        # The options are a list of strings in this form optname:optvalue
-        # We are storing all the options in a dict so we can use it later when
-        # generating the images
-        for opt in options:
-            if ":" not in opt:
-                continue
-            key, value = opt.split(":")
-            # if value is a number, convert it to the appropriate type
-            if is_float(value):
-                value = float(value)
-            elif is_int(value):
-                value = int(value)
-            elif value.lower() in ["true", "false"]:
-                value = value.lower() == "true"
-            self.options[key] = value
-
-        codec_repo = "neuphonic/neucodec"
-        if "codec_repo" in self.options:
-            codec_repo = self.options["codec_repo"]
-            del self.options["codec_repo"]
-        if "ref_text" in self.options:
-            self.ref_text = self.options["ref_text"]
-            del self.options["ref_text"]
-
-        self.AudioPath = None
-
-        if os.path.isabs(request.AudioPath):
-            self.AudioPath = request.AudioPath
-        elif request.AudioPath and request.ModelFile != "" and not os.path.isabs(request.AudioPath):
-            # get base path of modelFile
-            modelFileBase = os.path.dirname(request.ModelFile)
-            # modify LoraAdapter to be relative to modelFileBase
-            self.AudioPath = os.path.join(modelFileBase, request.AudioPath)
-        try:
-            print("Preparing models, please wait", file=sys.stderr)
-            self.model = NeuTTSAir(backbone_repo=request.Model, backbone_device=device, codec_repo=codec_repo, codec_device=device)
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-        # Implement your logic here for the LoadModel service
-        # Replace this with your desired response
-        return backend_pb2.Result(message="Model loaded successfully", success=True)
-
-    def TTS(self, request, context):
-        try:
-            kwargs = {}
-
-            # add options to kwargs
-            kwargs.update(self.options)
-
-            ref_codes = self.model.encode_reference(self.AudioPath)
-
-            wav = self.model.infer(request.text, ref_codes, self.ref_text)
-
-            sf.write(request.dst, wav, 24000)            
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-        return backend_pb2.Result(success=True)
-
-def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
-        options=[
-            ('grpc.max_message_length', 50 * 1024 * 1024),  # 50MB
-            ('grpc.max_send_message_length', 50 * 1024 * 1024),  # 50MB
-            ('grpc.max_receive_message_length', 50 * 1024 * 1024),  # 50MB
-        ])
-    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
-    server.add_insecure_port(address)
-    server.start()
-    print("Server started. Listening on: " + address, file=sys.stderr)
-
-    # Define the signal handler function
-    def signal_handler(sig, frame):
-        print("Received termination signal. Shutting down...")
-        server.stop(0)
-        sys.exit(0)
-
-    # Set the signal handlers for SIGINT and SIGTERM
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    try:
-        while True:
-            time.sleep(_ONE_DAY_IN_SECONDS)
-    except KeyboardInterrupt:
-        server.stop(0)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the gRPC server.")
-    parser.add_argument(
-        "--addr", default="localhost:50051", help="The address to bind the server to."
-    )
-    args = parser.parse_args()
-
-    serve(args.addr)

backend/python/neutts/install.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-set -e
-
-backend_dir=$(dirname $0)
-if [ -d $backend_dir/common ]; then
-    source $backend_dir/common/libbackend.sh
-else
-    source $backend_dir/../common/libbackend.sh
-fi
-
-# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links.
-# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match.
-# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index
-# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index
-if [ "x${BUILD_PROFILE}" == "xintel" ]; then
-    EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
-fi
-
-if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "xl4t" ]; then
-    export CMAKE_ARGS="-DGGML_CUDA=on"
-fi
-
-if [ "x${BUILD_TYPE}" == "xhipblas" ]; then
-    export CMAKE_ARGS="-DGGML_HIPBLAS=on"
-fi
-
-EXTRA_PIP_INSTALL_FLAGS+=" --no-build-isolation"
-
-git clone https://github.com/neuphonic/neutts-air neutts-air
-
-cp -rfv neutts-air/neuttsair ./
-
-installRequirements