feat(cli): benchmark configured text models (#12020)

* feat(cli): benchmark configured text models

Compare model aliases through a running LocalAI server. Report full
request latency and server-reported throughput with raw JSON samples.

Keep warmups separate and fail before writing results on request errors.

Assisted-by: Codex:GPT-6

* fix(cli): satisfy benchmark error checks

Explicitly discard errors from buffered report writes, HTTP response cleanup, and test server writes to pass errcheck without changing behavior.

Assisted-by: Codex:gpt-6 golangci-lint

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/cli/benchmark/benchmark.go

@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: MIT
+// Package benchmark measures text inference through a running LocalAI server.
+package benchmark
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"text/tabwriter"
+	"time"
+
+	"github.com/mudler/LocalAI/pkg/httpclient"
+)
+
+type Command struct {
+	Models    []string      `arg:"" required:"" help:"Configured text model names to benchmark sequentially."`
+	Endpoint  string        `default:"http://127.0.0.1:8080" help:"LocalAI server URL, optionally ending in /v1."`
+	APIKey    string        `name:"api-key" env:"LOCALAI_API_KEY,API_KEY" help:"API key for the server."`
+	Prompt    string        `default:"Explain why the sky is blue." help:"User prompt sent with every request."`
+	MaxTokens int           `default:"128" help:"Maximum completion tokens per request."`
+	Runs      int           `default:"3" help:"Measured requests per model."`
+	Warmup    int           `default:"1" help:"Unmeasured requests before each model's measured runs."`
+	Timeout   time.Duration `default:"5m" help:"Timeout for each request."`
+	JSON      bool          `name:"json" help:"Write settings and raw samples as JSON."`
+}
+
+type settings struct {
+	Endpoint    string  `json:"endpoint"`
+	Prompt      string  `json:"prompt"`
+	MaxTokens   int     `json:"max_tokens"`
+	Runs        int     `json:"runs"`
+	Warmup      int     `json:"warmup"`
+	Timeout     string  `json:"timeout"`
+	Temperature float64 `json:"temperature"`
+	Stream      bool    `json:"stream"`
+}
+type sample struct {
+	LatencySeconds   float64 `json:"latency_seconds"`
+	PromptTokens     *int    `json:"prompt_tokens"`
+	CompletionTokens *int    `json:"completion_tokens"`
+}
+type modelResult struct {
+	Model                     string   `json:"model"`
+	Samples                   []sample `json:"samples"`
+	MinSeconds                float64  `json:"min_seconds"`
+	MeanSeconds               float64  `json:"mean_seconds"`
+	MaxSeconds                float64  `json:"max_seconds"`
+	CompletionTokensPerSecond *float64 `json:"completion_tokens_per_second"`
+}
+type report struct {
+	Settings settings      `json:"settings"`
+	Results  []modelResult `json:"results"`
+}
+
+func (c *Command) Run() error {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	return c.run(ctx, os.Stdout)
+}
+
+func completionURL(endpoint string) (string, error) {
+	u, err := url.Parse(endpoint)
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(endpoint, "#") {
+		return "", errors.New("endpoint must be an HTTP(S) URL without credentials, query, or fragment")
+	}
+	path := strings.TrimRight(u.Path, "/")
+	if !strings.HasSuffix(path, "/v1") {
+		path += "/v1"
+	}
+	u.Path = path + "/chat/completions"
+	u.RawPath = ""
+	return u.String(), nil
+}
+
+func (c *Command) run(ctx context.Context, out io.Writer) error {
+	endpoint, err := completionURL(c.Endpoint)
+	if err != nil {
+		return err
+	}
+	if c.Runs <= 0 || c.Warmup < 0 || c.MaxTokens <= 0 || c.Timeout <= 0 {
+		return errors.New("runs, max-tokens, and timeout must be positive; warmup must be nonnegative")
+	}
+	if strings.TrimSpace(c.Prompt) == "" {
+		return errors.New("prompt must not be blank")
+	}
+	if len(c.Models) == 0 {
+		return errors.New("at least one model is required")
+	}
+	for _, model := range c.Models {
+		if strings.TrimSpace(model) == "" {
+			return errors.New("model names must not be blank")
+		}
+	}
+	client := httpclient.NewWithTimeout(c.Timeout)
+	defer client.CloseIdleConnections()
+	result := report{Settings: settings{Endpoint: endpoint, Prompt: c.Prompt, MaxTokens: c.MaxTokens, Runs: c.Runs, Warmup: c.Warmup, Timeout: c.Timeout.String()}}
+	for _, model := range c.Models {
+		measured := modelResult{Model: model}
+		for i := 0; i < c.Warmup; i++ {
+			if _, err := c.request(ctx, client, endpoint, model); err != nil {
+				return fmt.Errorf("model %q warmup %d: %w", model, i+1, err)
+			}
+		}
+		for i := 0; i < c.Runs; i++ {
+			s, err := c.request(ctx, client, endpoint, model)
+			if err != nil {
+				return fmt.Errorf("model %q run %d: %w", model, i+1, err)
+			}
+			measured.Samples = append(measured.Samples, s)
+		}
+		measured.summarize()
+		result.Results = append(result.Results, measured)
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	// Buffer the complete report so a failed model never leaves partial results.
+	var buffer bytes.Buffer
+	if c.JSON {
+		encoder := json.NewEncoder(&buffer)
+		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(result); err != nil {
+			return err
+		}
+	} else {
+		table := tabwriter.NewWriter(&buffer, 0, 4, 2, ' ', 0)
+		_, _ = fmt.Fprintln(table, "MODEL\tRUNS\tMIN (s)\tMEAN (s)\tMAX (s)\tEND-TO-END TOKENS/s")
+		for _, r := range result.Results {
+			throughput := "N/A"
+			if r.CompletionTokensPerSecond != nil {
+				throughput = fmt.Sprintf("%.2f", *r.CompletionTokensPerSecond)
+			}
+			_, _ = fmt.Fprintf(table, "%s\t%d\t%.4f\t%.4f\t%.4f\t%s\n", r.Model, len(r.Samples), r.MinSeconds, r.MeanSeconds, r.MaxSeconds, throughput)
+		}
+		if err := table.Flush(); err != nil {
+			return err
+		}
+	}
+	_, err = io.Copy(out, &buffer)
+	return err
+}
+
+func (c *Command) request(ctx context.Context, client *http.Client, endpoint, model string) (sample, error) {
+	var s sample
+	body, err := json.Marshal(struct {
+		Model       string              `json:"model"`
+		Messages    []map[string]string `json:"messages"`
+		MaxTokens   int                 `json:"max_tokens"`
+		Temperature float64             `json:"temperature"`
+		Stream      bool                `json:"stream"`
+	}{Model: model, Messages: []map[string]string{{"role": "user", "content": c.Prompt}}, MaxTokens: c.MaxTokens})
+	if err != nil {
+		return s, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
+	if err != nil {
+		return s, errors.New("cannot create benchmark request")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if c.APIKey != "" {
+		req.Header.Set("Authorization", "Bearer "+c.APIKey)
+	}
+	start := time.Now()
+	resp, err := client.Do(req)
+	if err != nil {
+		if ctx.Err() != nil {
+			return s, ctx.Err()
+		}
+		if errors.Is(err, context.DeadlineExceeded) {
+			return s, fmt.Errorf("request timed out: %w", context.DeadlineExceeded)
+		}
+		if errors.Is(err, httpclient.ErrRedirectBlocked) {
+			return s, httpclient.ErrRedirectBlocked
+		}
+		// Transport errors and server responses can echo credentials.
+		return s, errors.New("HTTP request failed")
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return s, fmt.Errorf("HTTP status %d", resp.StatusCode)
+	}
+	var response struct {
+		Choices []json.RawMessage `json:"choices"`
+		Usage   struct {
+			PromptTokens     *int `json:"prompt_tokens"`
+			CompletionTokens *int `json:"completion_tokens"`
+		} `json:"usage"`
+		Error json.RawMessage `json:"error"`
+	}
+	decoder := json.NewDecoder(resp.Body)
+	if err := decoder.Decode(&response); err != nil {
+		if ctx.Err() != nil {
+			return s, ctx.Err()
+		}
+		if errors.Is(err, context.DeadlineExceeded) {
+			return s, fmt.Errorf("request timed out: %w", context.DeadlineExceeded)
+		}
+		return s, errors.New("invalid JSON response")
+	}
+	var extra any
+	if err := decoder.Decode(&extra); err != io.EOF {
+		return s, errors.New("invalid trailing response data")
+	}
+	if len(response.Error) > 0 && string(response.Error) != "null" {
+		var detail struct {
+			Message string `json:"message"`
+		}
+		if json.Unmarshal(response.Error, &detail) == nil && detail.Message != "" {
+			message := detail.Message
+			if c.APIKey != "" {
+				message = strings.ReplaceAll(message, c.APIKey, "[redacted]")
+			}
+			return s, fmt.Errorf("server returned an API error: %s", message)
+		}
+		return s, errors.New("server returned an API error")
+	}
+	if len(response.Choices) == 0 {
+		return s, errors.New("response contains no choices")
+	}
+	s.LatencySeconds = time.Since(start).Seconds()
+	s.PromptTokens = response.Usage.PromptTokens
+	s.CompletionTokens = response.Usage.CompletionTokens
+	if (s.PromptTokens != nil && *s.PromptTokens < 0) || (s.CompletionTokens != nil && *s.CompletionTokens < 0) {
+		return s, errors.New("response contains negative token counts")
+	}
+	return s, nil
+}
+
+func (r *modelResult) summarize() {
+	r.MinSeconds = r.Samples[0].LatencySeconds
+	var seconds, tokens float64
+	available := true
+	for _, s := range r.Samples {
+		seconds += s.LatencySeconds
+		r.MinSeconds = min(r.MinSeconds, s.LatencySeconds)
+		r.MaxSeconds = max(r.MaxSeconds, s.LatencySeconds)
+		if s.CompletionTokens == nil {
+			available = false
+		} else {
+			tokens += float64(*s.CompletionTokens)
+		}
+	}
+	r.MeanSeconds = seconds / float64(len(r.Samples))
+	if available && seconds > 0 {
+		rate := tokens / seconds
+		r.CompletionTokensPerSecond = &rate
+	}
+}

core/cli/benchmark/benchmark_test.go

@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: MIT
+package benchmark
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/alecthomas/kong"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestBenchmark(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Benchmark") }
+
+var _ = Describe("Benchmark command", func() {
+	var cmd Command
+	var output bytes.Buffer
+	BeforeEach(func() {
+		cmd = Command{Models: []string{"a"}, Endpoint: "http://127.0.0.1:8080", Prompt: "hello", MaxTokens: 128, Runs: 2, Warmup: 1, Timeout: time.Second, JSON: true}
+		output.Reset()
+	})
+	It("parses required models and defaults", func() {
+		var c Command
+		parser, err := kong.New(&c)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = parser.Parse(nil)
+		Expect(err).To(HaveOccurred())
+		_, err = parser.Parse([]string{"a", "b"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(c.Models).To(Equal([]string{"a", "b"}))
+		Expect(c.Endpoint).To(Equal("http://127.0.0.1:8080"))
+		Expect(c.Runs).To(Equal(3))
+		Expect(c.Warmup).To(Equal(1))
+		Expect(c.MaxTokens).To(Equal(128))
+		Expect(c.Timeout).To(Equal(5 * time.Minute))
+		Expect(c.Prompt).NotTo(BeEmpty())
+	})
+	It("reads API key environment variables in priority order", func() {
+		for _, key := range []string{"LOCALAI_API_KEY", "API_KEY"} {
+			value, present := os.LookupEnv(key)
+			DeferCleanup(func() {
+				if present {
+					Expect(os.Setenv(key, value)).To(Succeed())
+				} else {
+					Expect(os.Unsetenv(key)).To(Succeed())
+				}
+			})
+		}
+		Expect(os.Unsetenv("LOCALAI_API_KEY")).To(Succeed())
+		Expect(os.Setenv("API_KEY", "fallback")).To(Succeed())
+		var c Command
+		parser, err := kong.New(&c)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = parser.Parse([]string{"a"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(c.APIKey).To(Equal("fallback"))
+		Expect(os.Setenv("LOCALAI_API_KEY", "preferred")).To(Succeed())
+		_, err = parser.Parse([]string{"a"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(c.APIKey).To(Equal("preferred"))
+	})
+	DescribeTable("normalizes endpoints", func(input, expected string) {
+		actual, err := completionURL(input)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(actual).To(Equal(expected))
+	},
+		Entry("root", "http://localhost:8080", "http://localhost:8080/v1/chat/completions"), Entry("slash", "http://localhost:8080/", "http://localhost:8080/v1/chat/completions"), Entry("v1", "https://example.org/v1/", "https://example.org/v1/chat/completions"), Entry("proxy", "https://example.org/proxy/", "https://example.org/proxy/v1/chat/completions"), Entry("proxy v1", "https://example.org/proxy/v1", "https://example.org/proxy/v1/chat/completions"))
+	It("posts authenticated requests sequentially and excludes each model's warmup", func() {
+		var models []string
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer GinkgoRecover()
+			Expect(r.Method).To(Equal("POST"))
+			Expect(r.URL.Path).To(Equal("/proxy/v1/chat/completions"))
+			Expect(r.Header.Get("Authorization")).To(Equal("Bearer secret"))
+			Expect(r.Header.Get("Content-Type")).To(Equal("application/json"))
+			var body map[string]any
+			Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed())
+			Expect(body["temperature"]).To(Equal(float64(0)))
+			Expect(body["stream"]).To(BeFalse())
+			Expect(body["max_tokens"]).To(Equal(float64(128)))
+			Expect(body["messages"]).To(Equal([]any{map[string]any{"role": "user", "content": "hello"}}))
+			models = append(models, body["model"].(string))
+			_, _ = fmt.Fprintf(w, `{"choices":[{}],"usage":{"prompt_tokens":5,"completion_tokens":%d}}`, len(models))
+		}))
+		defer server.Close()
+		cmd.Endpoint = server.URL + "/proxy"
+		cmd.APIKey = "secret"
+		cmd.Models = []string{"a", "b"}
+		Expect(cmd.run(context.Background(), &output)).To(Succeed())
+		Expect(models).To(Equal([]string{"a", "a", "a", "b", "b", "b"}))
+		Expect(output.String()).NotTo(ContainSubstring("secret"))
+		var result report
+		Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed())
+		Expect(result.Results).To(HaveLen(2))
+		Expect(result.Settings.Runs).To(Equal(2))
+		Expect(result.Settings.Warmup).To(Equal(1))
+		Expect(result.Settings.Prompt).To(Equal("hello"))
+		Expect(result.Settings.Temperature).To(BeZero())
+		Expect(result.Settings.Stream).To(BeFalse())
+		first := result.Results[0]
+		Expect(first.Samples).To(HaveLen(2))
+		Expect(*first.Samples[0].CompletionTokens).To(Equal(2))
+		Expect(*first.Samples[1].CompletionTokens).To(Equal(3))
+		Expect(*first.Samples[0].PromptTokens).To(Equal(5))
+		Expect(first.MinSeconds).To(BeNumerically(">", 0))
+		Expect(first.MeanSeconds).To(BeNumerically(">=", first.MinSeconds))
+		Expect(first.MaxSeconds).To(BeNumerically(">=", first.MeanSeconds))
+		Expect(*first.CompletionTokensPerSecond).To(BeNumerically("~", 5/(first.Samples[0].LatencySeconds+first.Samples[1].LatencySeconds), 0.001))
+	})
+	DescribeTable("preserves missing and zero usage", func(usage string, available bool) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprint(w, `{"choices":[{}]`+usage+`}`) }))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		cmd.Warmup = 0
+		Expect(cmd.run(context.Background(), &output)).To(Succeed())
+		var result report
+		Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed())
+		if available {
+			Expect(*result.Results[0].CompletionTokensPerSecond).To(BeZero())
+		} else {
+			Expect(result.Results[0].CompletionTokensPerSecond).To(BeNil())
+		}
+		cmd.JSON = false
+		output.Reset()
+		Expect(cmd.run(context.Background(), &output)).To(Succeed())
+		if !available {
+			Expect(output.String()).To(ContainSubstring("N/A"))
+		}
+	}, Entry("absent", "", false), Entry("empty", `,"usage":{}`, false), Entry("partial", `,"usage":{"prompt_tokens":0}`, false), Entry("zero", `,"usage":{"prompt_tokens":0,"completion_tokens":0}`, true))
+	It("retains API error details while redacting the key", func() {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = fmt.Fprint(w, `{"error":{"message":"model unavailable: secret"}}`)
+		}))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		cmd.APIKey = "secret"
+		cmd.Warmup = 0
+		err := cmd.run(context.Background(), &output)
+		Expect(err).To(MatchError(ContainSubstring(`model "a" run 1: server returned an API error: model unavailable: [redacted]`)))
+		Expect(output.Len()).To(BeZero())
+	})
+	It("marks throughput unavailable when one measured request omits usage", func() {
+		requests := 0
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			requests++
+			if requests == 1 {
+				_, _ = fmt.Fprint(w, `{"choices":[{}],"usage":{"completion_tokens":2}}`)
+			} else {
+				_, _ = fmt.Fprint(w, `{"choices":[{}]}`)
+			}
+		}))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		cmd.Warmup = 0
+		Expect(cmd.run(context.Background(), &output)).To(Succeed())
+		var result report
+		Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed())
+		Expect(result.Results[0].CompletionTokensPerSecond).To(BeNil())
+		Expect(result.Results[0].Samples[1].CompletionTokens).To(BeNil())
+	})
+	DescribeTable("fails without result output", func(status int, body string) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status); _, _ = fmt.Fprint(w, body) }))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		cmd.APIKey = "secret"
+		err := cmd.run(context.Background(), &output)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring(`model "a" warmup 1`))
+		Expect(err.Error()).NotTo(ContainSubstring("secret"))
+		Expect(output.Len()).To(BeZero())
+	}, Entry("HTTP", 500, `secret`), Entry("API", 200, `{"error":{"message":"secret"}}`), Entry("JSON", 200, `invalid`), Entry("empty choices", 200, `{"choices":[]}`), Entry("trailing JSON", 200, `{"choices":[{}]} {}`), Entry("negative tokens", 200, `{"choices":[{}],"usage":{"completion_tokens":-1}}`))
+	It("refuses redirects", func() {
+		reached := false
+		target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached = true }))
+		defer target.Close()
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect)
+		}))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		Expect(cmd.run(context.Background(), &output)).NotTo(Succeed())
+		Expect(reached).To(BeFalse())
+		Expect(output.Len()).To(BeZero())
+	})
+	It("honors cancellation", func() {
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+		err := cmd.run(ctx, &output)
+		Expect(err).To(MatchError(ContainSubstring("context canceled")))
+		Expect(output.Len()).To(BeZero())
+	})
+	It("times out requests", func() {
+		release := make(chan struct{})
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-release }))
+		defer server.Close()
+		defer close(release)
+		cmd.Endpoint = server.URL
+		cmd.Timeout = 20 * time.Millisecond
+		Expect(cmd.run(context.Background(), &output)).NotTo(Succeed())
+		Expect(output.Len()).To(BeZero())
+	})
+	It("times out while reading a response body", func() {
+		release := make(chan struct{})
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = fmt.Fprint(w, `{"choices":[`)
+			w.(http.Flusher).Flush()
+			<-release
+		}))
+		defer server.Close()
+		defer close(release)
+		cmd.Endpoint = server.URL
+		cmd.Timeout = 20 * time.Millisecond
+		Expect(cmd.run(context.Background(), &output)).To(MatchError(ContainSubstring("request timed out")))
+		Expect(output.Len()).To(BeZero())
+	})
+	It("cancels an active request", func() {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { cancel() }))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		Expect(cmd.run(ctx, &output)).To(MatchError(ContainSubstring("context canceled")))
+		Expect(output.Len()).To(BeZero())
+	})
+	DescribeTable("rejects invalid inputs before requests", func(change func(*Command)) {
+		reached := false
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached = true }))
+		defer server.Close()
+		cmd.Endpoint = server.URL
+		change(&cmd)
+		Expect(cmd.run(context.Background(), &output)).NotTo(Succeed())
+		Expect(reached).To(BeFalse())
+		Expect(output.Len()).To(BeZero())
+	}, Entry("runs", func(c *Command) { c.Runs = 0 }), Entry("warmup", func(c *Command) { c.Warmup = -1 }), Entry("tokens", func(c *Command) { c.MaxTokens = 0 }), Entry("timeout", func(c *Command) { c.Timeout = 0 }), Entry("prompt", func(c *Command) { c.Prompt = " " }), Entry("models", func(c *Command) { c.Models = nil }), Entry("blank model", func(c *Command) { c.Models = []string{"a", " "} }), Entry("scheme", func(c *Command) { c.Endpoint = "file:///tmp" }), Entry("host", func(c *Command) { c.Endpoint = "http:///v1" }), Entry("userinfo", func(c *Command) { c.Endpoint = "http://secret@localhost" }), Entry("query", func(c *Command) { c.Endpoint += "?secret" }), Entry("fragment", func(c *Command) { c.Endpoint += "#secret" }))
+})

core/cli/cli.go

@@ -1,12 +1,14 @@
 package cli
 
 import (
+	"github.com/mudler/LocalAI/core/cli/benchmark"
 	cliContext "github.com/mudler/LocalAI/core/cli/context"
 	"github.com/mudler/LocalAI/core/cli/worker"
 )
 
 var CLI struct {
 	cliContext.Context `embed:""`
+	Benchmark          benchmark.Command `cmd:"" help:"Benchmark configured text models against a running LocalAI server"`
 
 	Run             RunCMD             `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"`
 	Chat            ChatCMD            `cmd:"" help:"Run the built-in terminal agent against a LocalAI server"`

docs/content/features/benchmark.md

@@ -0,0 +1,84 @@
++++
+title = "Benchmark text models"
+weight = 35
++++
+
+Use `local-ai benchmark` to compare text inference through a running LocalAI
+server. Install and configure the models first. The command sends sequential,
+non-streaming requests to `/v1/chat/completions` and reports full request latency
+and end-to-end completion tokens per second.
+
+Pass one or more configured model names. To compare backends, configure separate
+model aliases with the desired backend, then pass those aliases. Backend names
+alone are not model names. The command does not install, discover, or unload
+models; the server's normal loading and eviction settings still apply.
+
+For example, with models named `text-llama-cpp` and `text-vllm` configured:
+
+```sh
+local-ai benchmark text-llama-cpp text-vllm --runs 5 --warmup 1 \
+  --prompt 'Explain how a rainbow forms.' --max-tokens 128
+```
+
+To save settings, per-model summaries, and every measured sample:
+
+```sh
+local-ai benchmark text-llama-cpp text-vllm --json > benchmark.json
+```
+
+For an authenticated server, set `LOCALAI_API_KEY` or `API_KEY` in the environment.
+The API key is excluded from the JSON report. Use `--endpoint` for a remote server
+or reverse proxy:
+
+```sh
+local-ai benchmark text-llama-cpp --endpoint https://localai.example.org/proxy/v1
+```
+
+The endpoint accepts a server root, an optional `/v1` suffix, and trailing
+slashes. A reverse proxy path prefix is preserved. Redirects are refused.
+Credentials in the URL, query strings, and fragments are rejected.
+
+## Arguments and flags
+
+| Argument or flag | Default | Description |
+|---|---|---|
+| `MODEL ...` | Required | One or more configured text model names. |
+| `--endpoint` | `http://127.0.0.1:8080` | Server URL, optionally ending in `/v1`. |
+| `--api-key` | Unset | API key; also reads `LOCALAI_API_KEY`, then `API_KEY`. |
+| `--prompt` | `Explain why the sky is blue.` | Nonblank user message repeated for every request. |
+| `--max-tokens` | `128` | Positive maximum number of completion tokens per request. |
+| `--runs` | `3` | Positive number of measured requests per model. |
+| `--warmup` | `1` | Unmeasured requests before each model; zero disables warmups. |
+| `--timeout` | `5m` | Positive timeout per request, including reading its response. |
+| `--json` | `false` | Write JSON instead of a table. |
+| `-h`, `--help` | | Show command help. |
+
+Every request sets temperature to `0` and streaming to `false`. Interrupting the
+command cancels the active request. A failed request stops the benchmark with a
+model and run error; results are written only after every model succeeds.
+
+## Reading the results
+
+Each model has minimum, mean, and maximum latency across measured requests.
+Latency runs from sending the request through parsing the complete response.
+It includes transport, queueing, prompt processing, generation, and response
+parsing. This command does not measure time to first token.
+
+End-to-end completion tokens per second is the sum of server-reported completion
+tokens divided by the sum of full request durations. It is not decode-only speed
+or a substitute for `llama-bench` kernel measurements. If any measured response
+omits completion token usage, throughput is `null` in JSON and `N/A` in the table.
+Reported zero tokens remain zero. Missing prompt or completion counts remain
+`null` in each JSON sample; the command never estimates tokens from text length.
+
+Warmups run separately for each model and do not appear in measurements. They can
+absorb model loading time, but repeated prompts can also benefit from prompt
+caching. With `--warmup 0`, measured requests can include model loading. Other
+clients and server queueing can affect results; compare under similar load.
+
+Keep hardware, quantization, context size, backend settings, and prompt consistent
+when comparing engines. Different model tokenizers can report different token
+counts for the same text, and models can stop before `--max-tokens`. Temperature
+zero does not guarantee identical output across models or engines. The JSON
+settings describe the benchmark requests, not the server's full model
+configuration; record that configuration alongside the report.

docs/content/reference/cli-reference.md

@@ -7,6 +7,8 @@ url = '/reference/cli-reference'
 
 Complete reference for all LocalAI command-line interface (CLI) parameters and environment variables.
 
+For client-side text inference measurements, see [Benchmark text models]({{% relref "features/benchmark" %}}).
+
 These options configure the LocalAI server process. To configure an individual
 model, see [Model Configuration]({{% relref "advanced/model-configuration" %}}).