From 35819d9e0de06f90fb3ff3e6616f0b7da13ffa67 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 19:06:43 +0200 Subject: [PATCH] feat(cli): benchmark configured text models (#12020) * feat(cli): benchmark configured text models Compare model aliases through a running LocalAI server. Report full request latency and server-reported throughput with raw JSON samples. Keep warmups separate and fail before writing results on request errors. Assisted-by: Codex:GPT-6 * fix(cli): satisfy benchmark error checks Explicitly discard errors from buffered report writes, HTTP response cleanup, and test server writes to pass errcheck without changing behavior. Assisted-by: Codex:gpt-6 golangci-lint --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/cli/benchmark/benchmark.go | 257 ++++++++++++++++++++++++ core/cli/benchmark/benchmark_test.go | 242 ++++++++++++++++++++++ core/cli/cli.go | 2 + docs/content/features/benchmark.md | 84 ++++++++ docs/content/reference/cli-reference.md | 2 + 5 files changed, 587 insertions(+) create mode 100644 core/cli/benchmark/benchmark.go create mode 100644 core/cli/benchmark/benchmark_test.go create mode 100644 docs/content/features/benchmark.md diff --git a/core/cli/benchmark/benchmark.go b/core/cli/benchmark/benchmark.go new file mode 100644 index 000000000..0ceef79f8 --- /dev/null +++ b/core/cli/benchmark/benchmark.go @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: MIT +// Package benchmark measures text inference through a running LocalAI server. +package benchmark + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "text/tabwriter" + "time" + + "github.com/mudler/LocalAI/pkg/httpclient" +) + +type Command struct { + Models []string `arg:"" required:"" help:"Configured text model names to benchmark sequentially."` + Endpoint string `default:"http://127.0.0.1:8080" help:"LocalAI server URL, optionally ending in /v1."` + APIKey string `name:"api-key" env:"LOCALAI_API_KEY,API_KEY" help:"API key for the server."` + Prompt string `default:"Explain why the sky is blue." help:"User prompt sent with every request."` + MaxTokens int `default:"128" help:"Maximum completion tokens per request."` + Runs int `default:"3" help:"Measured requests per model."` + Warmup int `default:"1" help:"Unmeasured requests before each model's measured runs."` + Timeout time.Duration `default:"5m" help:"Timeout for each request."` + JSON bool `name:"json" help:"Write settings and raw samples as JSON."` +} + +type settings struct { + Endpoint string `json:"endpoint"` + Prompt string `json:"prompt"` + MaxTokens int `json:"max_tokens"` + Runs int `json:"runs"` + Warmup int `json:"warmup"` + Timeout string `json:"timeout"` + Temperature float64 `json:"temperature"` + Stream bool `json:"stream"` +} +type sample struct { + LatencySeconds float64 `json:"latency_seconds"` + PromptTokens *int `json:"prompt_tokens"` + CompletionTokens *int `json:"completion_tokens"` +} +type modelResult struct { + Model string `json:"model"` + Samples []sample `json:"samples"` + MinSeconds float64 `json:"min_seconds"` + MeanSeconds float64 `json:"mean_seconds"` + MaxSeconds float64 `json:"max_seconds"` + CompletionTokensPerSecond *float64 `json:"completion_tokens_per_second"` +} +type report struct { + Settings settings `json:"settings"` + Results []modelResult `json:"results"` +} + +func (c *Command) Run() error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + return c.run(ctx, os.Stdout) +} + +func completionURL(endpoint string) (string, error) { + u, err := url.Parse(endpoint) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(endpoint, "#") { + return "", errors.New("endpoint must be an HTTP(S) URL without credentials, query, or fragment") + } + path := strings.TrimRight(u.Path, "/") + if !strings.HasSuffix(path, "/v1") { + path += "/v1" + } + u.Path = path + "/chat/completions" + u.RawPath = "" + return u.String(), nil +} + +func (c *Command) run(ctx context.Context, out io.Writer) error { + endpoint, err := completionURL(c.Endpoint) + if err != nil { + return err + } + if c.Runs <= 0 || c.Warmup < 0 || c.MaxTokens <= 0 || c.Timeout <= 0 { + return errors.New("runs, max-tokens, and timeout must be positive; warmup must be nonnegative") + } + if strings.TrimSpace(c.Prompt) == "" { + return errors.New("prompt must not be blank") + } + if len(c.Models) == 0 { + return errors.New("at least one model is required") + } + for _, model := range c.Models { + if strings.TrimSpace(model) == "" { + return errors.New("model names must not be blank") + } + } + client := httpclient.NewWithTimeout(c.Timeout) + defer client.CloseIdleConnections() + result := report{Settings: settings{Endpoint: endpoint, Prompt: c.Prompt, MaxTokens: c.MaxTokens, Runs: c.Runs, Warmup: c.Warmup, Timeout: c.Timeout.String()}} + for _, model := range c.Models { + measured := modelResult{Model: model} + for i := 0; i < c.Warmup; i++ { + if _, err := c.request(ctx, client, endpoint, model); err != nil { + return fmt.Errorf("model %q warmup %d: %w", model, i+1, err) + } + } + for i := 0; i < c.Runs; i++ { + s, err := c.request(ctx, client, endpoint, model) + if err != nil { + return fmt.Errorf("model %q run %d: %w", model, i+1, err) + } + measured.Samples = append(measured.Samples, s) + } + measured.summarize() + result.Results = append(result.Results, measured) + } + if err := ctx.Err(); err != nil { + return err + } + // Buffer the complete report so a failed model never leaves partial results. + var buffer bytes.Buffer + if c.JSON { + encoder := json.NewEncoder(&buffer) + encoder.SetIndent("", " ") + if err := encoder.Encode(result); err != nil { + return err + } + } else { + table := tabwriter.NewWriter(&buffer, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(table, "MODEL\tRUNS\tMIN (s)\tMEAN (s)\tMAX (s)\tEND-TO-END TOKENS/s") + for _, r := range result.Results { + throughput := "N/A" + if r.CompletionTokensPerSecond != nil { + throughput = fmt.Sprintf("%.2f", *r.CompletionTokensPerSecond) + } + _, _ = fmt.Fprintf(table, "%s\t%d\t%.4f\t%.4f\t%.4f\t%s\n", r.Model, len(r.Samples), r.MinSeconds, r.MeanSeconds, r.MaxSeconds, throughput) + } + if err := table.Flush(); err != nil { + return err + } + } + _, err = io.Copy(out, &buffer) + return err +} + +func (c *Command) request(ctx context.Context, client *http.Client, endpoint, model string) (sample, error) { + var s sample + body, err := json.Marshal(struct { + Model string `json:"model"` + Messages []map[string]string `json:"messages"` + MaxTokens int `json:"max_tokens"` + Temperature float64 `json:"temperature"` + Stream bool `json:"stream"` + }{Model: model, Messages: []map[string]string{{"role": "user", "content": c.Prompt}}, MaxTokens: c.MaxTokens}) + if err != nil { + return s, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return s, errors.New("cannot create benchmark request") + } + req.Header.Set("Content-Type", "application/json") + if c.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+c.APIKey) + } + start := time.Now() + resp, err := client.Do(req) + if err != nil { + if ctx.Err() != nil { + return s, ctx.Err() + } + if errors.Is(err, context.DeadlineExceeded) { + return s, fmt.Errorf("request timed out: %w", context.DeadlineExceeded) + } + if errors.Is(err, httpclient.ErrRedirectBlocked) { + return s, httpclient.ErrRedirectBlocked + } + // Transport errors and server responses can echo credentials. + return s, errors.New("HTTP request failed") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return s, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + var response struct { + Choices []json.RawMessage `json:"choices"` + Usage struct { + PromptTokens *int `json:"prompt_tokens"` + CompletionTokens *int `json:"completion_tokens"` + } `json:"usage"` + Error json.RawMessage `json:"error"` + } + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&response); err != nil { + if ctx.Err() != nil { + return s, ctx.Err() + } + if errors.Is(err, context.DeadlineExceeded) { + return s, fmt.Errorf("request timed out: %w", context.DeadlineExceeded) + } + return s, errors.New("invalid JSON response") + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return s, errors.New("invalid trailing response data") + } + if len(response.Error) > 0 && string(response.Error) != "null" { + var detail struct { + Message string `json:"message"` + } + if json.Unmarshal(response.Error, &detail) == nil && detail.Message != "" { + message := detail.Message + if c.APIKey != "" { + message = strings.ReplaceAll(message, c.APIKey, "[redacted]") + } + return s, fmt.Errorf("server returned an API error: %s", message) + } + return s, errors.New("server returned an API error") + } + if len(response.Choices) == 0 { + return s, errors.New("response contains no choices") + } + s.LatencySeconds = time.Since(start).Seconds() + s.PromptTokens = response.Usage.PromptTokens + s.CompletionTokens = response.Usage.CompletionTokens + if (s.PromptTokens != nil && *s.PromptTokens < 0) || (s.CompletionTokens != nil && *s.CompletionTokens < 0) { + return s, errors.New("response contains negative token counts") + } + return s, nil +} + +func (r *modelResult) summarize() { + r.MinSeconds = r.Samples[0].LatencySeconds + var seconds, tokens float64 + available := true + for _, s := range r.Samples { + seconds += s.LatencySeconds + r.MinSeconds = min(r.MinSeconds, s.LatencySeconds) + r.MaxSeconds = max(r.MaxSeconds, s.LatencySeconds) + if s.CompletionTokens == nil { + available = false + } else { + tokens += float64(*s.CompletionTokens) + } + } + r.MeanSeconds = seconds / float64(len(r.Samples)) + if available && seconds > 0 { + rate := tokens / seconds + r.CompletionTokensPerSecond = &rate + } +} diff --git a/core/cli/benchmark/benchmark_test.go b/core/cli/benchmark/benchmark_test.go new file mode 100644 index 000000000..fcd007f49 --- /dev/null +++ b/core/cli/benchmark/benchmark_test.go @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: MIT +package benchmark + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/alecthomas/kong" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBenchmark(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Benchmark") } + +var _ = Describe("Benchmark command", func() { + var cmd Command + var output bytes.Buffer + BeforeEach(func() { + cmd = Command{Models: []string{"a"}, Endpoint: "http://127.0.0.1:8080", Prompt: "hello", MaxTokens: 128, Runs: 2, Warmup: 1, Timeout: time.Second, JSON: true} + output.Reset() + }) + It("parses required models and defaults", func() { + var c Command + parser, err := kong.New(&c) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse(nil) + Expect(err).To(HaveOccurred()) + _, err = parser.Parse([]string{"a", "b"}) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Models).To(Equal([]string{"a", "b"})) + Expect(c.Endpoint).To(Equal("http://127.0.0.1:8080")) + Expect(c.Runs).To(Equal(3)) + Expect(c.Warmup).To(Equal(1)) + Expect(c.MaxTokens).To(Equal(128)) + Expect(c.Timeout).To(Equal(5 * time.Minute)) + Expect(c.Prompt).NotTo(BeEmpty()) + }) + It("reads API key environment variables in priority order", func() { + for _, key := range []string{"LOCALAI_API_KEY", "API_KEY"} { + value, present := os.LookupEnv(key) + DeferCleanup(func() { + if present { + Expect(os.Setenv(key, value)).To(Succeed()) + } else { + Expect(os.Unsetenv(key)).To(Succeed()) + } + }) + } + Expect(os.Unsetenv("LOCALAI_API_KEY")).To(Succeed()) + Expect(os.Setenv("API_KEY", "fallback")).To(Succeed()) + var c Command + parser, err := kong.New(&c) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse([]string{"a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(c.APIKey).To(Equal("fallback")) + Expect(os.Setenv("LOCALAI_API_KEY", "preferred")).To(Succeed()) + _, err = parser.Parse([]string{"a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(c.APIKey).To(Equal("preferred")) + }) + DescribeTable("normalizes endpoints", func(input, expected string) { + actual, err := completionURL(input) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(expected)) + }, + Entry("root", "http://localhost:8080", "http://localhost:8080/v1/chat/completions"), Entry("slash", "http://localhost:8080/", "http://localhost:8080/v1/chat/completions"), Entry("v1", "https://example.org/v1/", "https://example.org/v1/chat/completions"), Entry("proxy", "https://example.org/proxy/", "https://example.org/proxy/v1/chat/completions"), Entry("proxy v1", "https://example.org/proxy/v1", "https://example.org/proxy/v1/chat/completions")) + It("posts authenticated requests sequentially and excludes each model's warmup", func() { + var models []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + Expect(r.Method).To(Equal("POST")) + Expect(r.URL.Path).To(Equal("/proxy/v1/chat/completions")) + Expect(r.Header.Get("Authorization")).To(Equal("Bearer secret")) + Expect(r.Header.Get("Content-Type")).To(Equal("application/json")) + var body map[string]any + Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) + Expect(body["temperature"]).To(Equal(float64(0))) + Expect(body["stream"]).To(BeFalse()) + Expect(body["max_tokens"]).To(Equal(float64(128))) + Expect(body["messages"]).To(Equal([]any{map[string]any{"role": "user", "content": "hello"}})) + models = append(models, body["model"].(string)) + _, _ = fmt.Fprintf(w, `{"choices":[{}],"usage":{"prompt_tokens":5,"completion_tokens":%d}}`, len(models)) + })) + defer server.Close() + cmd.Endpoint = server.URL + "/proxy" + cmd.APIKey = "secret" + cmd.Models = []string{"a", "b"} + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + Expect(models).To(Equal([]string{"a", "a", "a", "b", "b", "b"})) + Expect(output.String()).NotTo(ContainSubstring("secret")) + var result report + Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed()) + Expect(result.Results).To(HaveLen(2)) + Expect(result.Settings.Runs).To(Equal(2)) + Expect(result.Settings.Warmup).To(Equal(1)) + Expect(result.Settings.Prompt).To(Equal("hello")) + Expect(result.Settings.Temperature).To(BeZero()) + Expect(result.Settings.Stream).To(BeFalse()) + first := result.Results[0] + Expect(first.Samples).To(HaveLen(2)) + Expect(*first.Samples[0].CompletionTokens).To(Equal(2)) + Expect(*first.Samples[1].CompletionTokens).To(Equal(3)) + Expect(*first.Samples[0].PromptTokens).To(Equal(5)) + Expect(first.MinSeconds).To(BeNumerically(">", 0)) + Expect(first.MeanSeconds).To(BeNumerically(">=", first.MinSeconds)) + Expect(first.MaxSeconds).To(BeNumerically(">=", first.MeanSeconds)) + Expect(*first.CompletionTokensPerSecond).To(BeNumerically("~", 5/(first.Samples[0].LatencySeconds+first.Samples[1].LatencySeconds), 0.001)) + }) + DescribeTable("preserves missing and zero usage", func(usage string, available bool) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprint(w, `{"choices":[{}]`+usage+`}`) })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.Warmup = 0 + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + var result report + Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed()) + if available { + Expect(*result.Results[0].CompletionTokensPerSecond).To(BeZero()) + } else { + Expect(result.Results[0].CompletionTokensPerSecond).To(BeNil()) + } + cmd.JSON = false + output.Reset() + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + if !available { + Expect(output.String()).To(ContainSubstring("N/A")) + } + }, Entry("absent", "", false), Entry("empty", `,"usage":{}`, false), Entry("partial", `,"usage":{"prompt_tokens":0}`, false), Entry("zero", `,"usage":{"prompt_tokens":0,"completion_tokens":0}`, true)) + It("retains API error details while redacting the key", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"error":{"message":"model unavailable: secret"}}`) + })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.APIKey = "secret" + cmd.Warmup = 0 + err := cmd.run(context.Background(), &output) + Expect(err).To(MatchError(ContainSubstring(`model "a" run 1: server returned an API error: model unavailable: [redacted]`))) + Expect(output.Len()).To(BeZero()) + }) + It("marks throughput unavailable when one measured request omits usage", func() { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if requests == 1 { + _, _ = fmt.Fprint(w, `{"choices":[{}],"usage":{"completion_tokens":2}}`) + } else { + _, _ = fmt.Fprint(w, `{"choices":[{}]}`) + } + })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.Warmup = 0 + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + var result report + Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed()) + Expect(result.Results[0].CompletionTokensPerSecond).To(BeNil()) + Expect(result.Results[0].Samples[1].CompletionTokens).To(BeNil()) + }) + DescribeTable("fails without result output", func(status int, body string) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status); _, _ = fmt.Fprint(w, body) })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.APIKey = "secret" + err := cmd.run(context.Background(), &output) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`model "a" warmup 1`)) + Expect(err.Error()).NotTo(ContainSubstring("secret")) + Expect(output.Len()).To(BeZero()) + }, Entry("HTTP", 500, `secret`), Entry("API", 200, `{"error":{"message":"secret"}}`), Entry("JSON", 200, `invalid`), Entry("empty choices", 200, `{"choices":[]}`), Entry("trailing JSON", 200, `{"choices":[{}]} {}`), Entry("negative tokens", 200, `{"choices":[{}],"usage":{"completion_tokens":-1}}`)) + It("refuses redirects", func() { + reached := false + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached = true })) + defer target.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + cmd.Endpoint = server.URL + Expect(cmd.run(context.Background(), &output)).NotTo(Succeed()) + Expect(reached).To(BeFalse()) + Expect(output.Len()).To(BeZero()) + }) + It("honors cancellation", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := cmd.run(ctx, &output) + Expect(err).To(MatchError(ContainSubstring("context canceled"))) + Expect(output.Len()).To(BeZero()) + }) + It("times out requests", func() { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-release })) + defer server.Close() + defer close(release) + cmd.Endpoint = server.URL + cmd.Timeout = 20 * time.Millisecond + Expect(cmd.run(context.Background(), &output)).NotTo(Succeed()) + Expect(output.Len()).To(BeZero()) + }) + It("times out while reading a response body", func() { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"choices":[`) + w.(http.Flusher).Flush() + <-release + })) + defer server.Close() + defer close(release) + cmd.Endpoint = server.URL + cmd.Timeout = 20 * time.Millisecond + Expect(cmd.run(context.Background(), &output)).To(MatchError(ContainSubstring("request timed out"))) + Expect(output.Len()).To(BeZero()) + }) + It("cancels an active request", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { cancel() })) + defer server.Close() + cmd.Endpoint = server.URL + Expect(cmd.run(ctx, &output)).To(MatchError(ContainSubstring("context canceled"))) + Expect(output.Len()).To(BeZero()) + }) + DescribeTable("rejects invalid inputs before requests", func(change func(*Command)) { + reached := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached = true })) + defer server.Close() + cmd.Endpoint = server.URL + change(&cmd) + Expect(cmd.run(context.Background(), &output)).NotTo(Succeed()) + Expect(reached).To(BeFalse()) + Expect(output.Len()).To(BeZero()) + }, Entry("runs", func(c *Command) { c.Runs = 0 }), Entry("warmup", func(c *Command) { c.Warmup = -1 }), Entry("tokens", func(c *Command) { c.MaxTokens = 0 }), Entry("timeout", func(c *Command) { c.Timeout = 0 }), Entry("prompt", func(c *Command) { c.Prompt = " " }), Entry("models", func(c *Command) { c.Models = nil }), Entry("blank model", func(c *Command) { c.Models = []string{"a", " "} }), Entry("scheme", func(c *Command) { c.Endpoint = "file:///tmp" }), Entry("host", func(c *Command) { c.Endpoint = "http:///v1" }), Entry("userinfo", func(c *Command) { c.Endpoint = "http://secret@localhost" }), Entry("query", func(c *Command) { c.Endpoint += "?secret" }), Entry("fragment", func(c *Command) { c.Endpoint += "#secret" })) +}) diff --git a/core/cli/cli.go b/core/cli/cli.go index 77bf128cc..2b17ff96d 100644 --- a/core/cli/cli.go +++ b/core/cli/cli.go @@ -1,12 +1,14 @@ package cli import ( + "github.com/mudler/LocalAI/core/cli/benchmark" cliContext "github.com/mudler/LocalAI/core/cli/context" "github.com/mudler/LocalAI/core/cli/worker" ) var CLI struct { cliContext.Context `embed:""` + Benchmark benchmark.Command `cmd:"" help:"Benchmark configured text models against a running LocalAI server"` Run RunCMD `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"` Chat ChatCMD `cmd:"" help:"Run the built-in terminal agent against a LocalAI server"` diff --git a/docs/content/features/benchmark.md b/docs/content/features/benchmark.md new file mode 100644 index 000000000..f57a7783c --- /dev/null +++ b/docs/content/features/benchmark.md @@ -0,0 +1,84 @@ ++++ +title = "Benchmark text models" +weight = 35 ++++ + +Use `local-ai benchmark` to compare text inference through a running LocalAI +server. Install and configure the models first. The command sends sequential, +non-streaming requests to `/v1/chat/completions` and reports full request latency +and end-to-end completion tokens per second. + +Pass one or more configured model names. To compare backends, configure separate +model aliases with the desired backend, then pass those aliases. Backend names +alone are not model names. The command does not install, discover, or unload +models; the server's normal loading and eviction settings still apply. + +For example, with models named `text-llama-cpp` and `text-vllm` configured: + +```sh +local-ai benchmark text-llama-cpp text-vllm --runs 5 --warmup 1 \ + --prompt 'Explain how a rainbow forms.' --max-tokens 128 +``` + +To save settings, per-model summaries, and every measured sample: + +```sh +local-ai benchmark text-llama-cpp text-vllm --json > benchmark.json +``` + +For an authenticated server, set `LOCALAI_API_KEY` or `API_KEY` in the environment. +The API key is excluded from the JSON report. Use `--endpoint` for a remote server +or reverse proxy: + +```sh +local-ai benchmark text-llama-cpp --endpoint https://localai.example.org/proxy/v1 +``` + +The endpoint accepts a server root, an optional `/v1` suffix, and trailing +slashes. A reverse proxy path prefix is preserved. Redirects are refused. +Credentials in the URL, query strings, and fragments are rejected. + +## Arguments and flags + +| Argument or flag | Default | Description | +|---|---|---| +| `MODEL ...` | Required | One or more configured text model names. | +| `--endpoint` | `http://127.0.0.1:8080` | Server URL, optionally ending in `/v1`. | +| `--api-key` | Unset | API key; also reads `LOCALAI_API_KEY`, then `API_KEY`. | +| `--prompt` | `Explain why the sky is blue.` | Nonblank user message repeated for every request. | +| `--max-tokens` | `128` | Positive maximum number of completion tokens per request. | +| `--runs` | `3` | Positive number of measured requests per model. | +| `--warmup` | `1` | Unmeasured requests before each model; zero disables warmups. | +| `--timeout` | `5m` | Positive timeout per request, including reading its response. | +| `--json` | `false` | Write JSON instead of a table. | +| `-h`, `--help` | | Show command help. | + +Every request sets temperature to `0` and streaming to `false`. Interrupting the +command cancels the active request. A failed request stops the benchmark with a +model and run error; results are written only after every model succeeds. + +## Reading the results + +Each model has minimum, mean, and maximum latency across measured requests. +Latency runs from sending the request through parsing the complete response. +It includes transport, queueing, prompt processing, generation, and response +parsing. This command does not measure time to first token. + +End-to-end completion tokens per second is the sum of server-reported completion +tokens divided by the sum of full request durations. It is not decode-only speed +or a substitute for `llama-bench` kernel measurements. If any measured response +omits completion token usage, throughput is `null` in JSON and `N/A` in the table. +Reported zero tokens remain zero. Missing prompt or completion counts remain +`null` in each JSON sample; the command never estimates tokens from text length. + +Warmups run separately for each model and do not appear in measurements. They can +absorb model loading time, but repeated prompts can also benefit from prompt +caching. With `--warmup 0`, measured requests can include model loading. Other +clients and server queueing can affect results; compare under similar load. + +Keep hardware, quantization, context size, backend settings, and prompt consistent +when comparing engines. Different model tokenizers can report different token +counts for the same text, and models can stop before `--max-tokens`. Temperature +zero does not guarantee identical output across models or engines. The JSON +settings describe the benchmark requests, not the server's full model +configuration; record that configuration alongside the report. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index f91beb6a6..d8ad4a17d 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -7,6 +7,8 @@ url = '/reference/cli-reference' Complete reference for all LocalAI command-line interface (CLI) parameters and environment variables. +For client-side text inference measurements, see [Benchmark text models]({{% relref "features/benchmark" %}}). + These options configure the LocalAI server process. To configure an individual model, see [Model Configuration]({{% relref "advanced/model-configuration" %}}).