From aca800d003aced744d80d78df9cd728250d56472 Mon Sep 17 00:00:00 2001
From: manunicholasjacob <manunicholasjacob@gmail.com>
Date: Tue, 13 Oct 2026 22:56:25 -0500
Subject: [PATCH] fix(xsysinfo): cap the default thread count by the container
 CPU limit

When --threads / LOCALAI_THREADS is unset, LocalAI uses the host's
physical core count. Inside a container with a CPU limit (docker --cpus,
a Kubernetes limits.cpu) that starts one thread per host core inside a
smaller CFS quota, so the backend is throttled for most of each period.

Since Go 1.25 the default GOMAXPROCS already follows the affinity mask and
the cgroup CPU limit. Use the smaller of the physical core count and
GOMAXPROCS as the default. Hosts without a limit are unchanged.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: manunicholasjacob <manunicholasjacob@gmail.com>
---
 core/application/startup.go |  7 ++++---
 pkg/xsysinfo/cpu.go         | 22 ++++++++++++++++++++++
 pkg/xsysinfo/cpu_test.go    | 26 ++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/core/application/startup.go b/core/application/startup.go
index c0ddb0f31..c50be0d5e 100644
--- a/core/application/startup.go
+++ b/core/application/startup.go
@@ -56,10 +56,11 @@ func New(opts ...config.AppOption) (*Application, error) {
 	loadRuntimeSettingsFromFile(options)
 
 	// WithThreads no longer eagerly resolves 0 (so the settings merge can
-	// tell "unset" from "env-set"); resolve the physical-core default now
-	// that env, CLI, and file have all had their say.
+	// tell "unset" from "env-set"); resolve the default (physical cores,
+	// capped by the container CPU limit) now that env, CLI, and file have all
+	// had their say.
 	if options.Threads == 0 {
-		options.Threads = xsysinfo.CPUPhysicalCores()
+		options.Threads = xsysinfo.DefaultThreads()
 	}
 
 	trace.ConfigureBackendTracePersistence(options.DataPath)
diff --git a/pkg/xsysinfo/cpu.go b/pkg/xsysinfo/cpu.go
index db948a15b..b2282bbb3 100644
--- a/pkg/xsysinfo/cpu.go
+++ b/pkg/xsysinfo/cpu.go
@@ -3,6 +3,7 @@ package xsysinfo
 import (
 	"fmt"
 	"maps"
+	"runtime"
 	"slices"
 	"sort"
 
@@ -82,3 +83,24 @@ func CPUPhysicalCores() int {
 	}
 	return cpuid.CPU.PhysicalCores
 }
+
+// DefaultThreads returns the default number of inference threads: the number
+// of physical cores, capped by the number of CPUs the Go runtime may use.
+// Since Go 1.25 the default GOMAXPROCS follows the CPU affinity mask and the
+// cgroup CPU limit, so a container started with --cpus=2 or a Kubernetes CPU
+// limit of 2 gets 2 threads instead of one per physical core on the host.
+func DefaultThreads() int {
+	return capThreads(CPUPhysicalCores(), runtime.GOMAXPROCS(0))
+}
+
+// capThreads returns physical, lowered to usable when usable is smaller and
+// positive, and never less than 1.
+func capThreads(physical, usable int) int {
+	if physical < 1 {
+		physical = 1
+	}
+	if usable > 0 && usable < physical {
+		return usable
+	}
+	return physical
+}
diff --git a/pkg/xsysinfo/cpu_test.go b/pkg/xsysinfo/cpu_test.go
index a34e638ee..6be46427c 100644
--- a/pkg/xsysinfo/cpu_test.go
+++ b/pkg/xsysinfo/cpu_test.go
@@ -2,6 +2,7 @@ package xsysinfo
 
 import (
 	"math"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -18,3 +19,28 @@ var _ = Describe("CPU telemetry", func() {
 		Expect(math.IsNaN(info.Load1)).To(BeFalse())
 	})
 })
+
+var _ = Describe("Default thread count", func() {
+	It("keeps the physical core count when the runtime may use more CPUs", func() {
+		Expect(capThreads(8, 20)).To(Equal(8))
+		Expect(capThreads(8, 8)).To(Equal(8))
+	})
+
+	It("lowers the count to the CPUs the runtime may use", func() {
+		Expect(capThreads(14, 2)).To(Equal(2))
+		Expect(capThreads(14, 1)).To(Equal(1))
+	})
+
+	It("ignores a non-positive limit and never returns less than 1", func() {
+		Expect(capThreads(8, 0)).To(Equal(8))
+		Expect(capThreads(0, 0)).To(Equal(1))
+		Expect(capThreads(0, 4)).To(Equal(1))
+	})
+
+	It("follows GOMAXPROCS", func() {
+		prev := runtime.GOMAXPROCS(1)
+		DeferCleanup(func() { runtime.GOMAXPROCS(prev) })
+
+		Expect(DefaultThreads()).To(Equal(1))
+	})
+})