diff --git a/core/cli/run.go b/core/cli/run.go index 600b2a997..749dd16f0 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -243,6 +243,23 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { return nil } + activatedListeners, err := systemdActivatedListeners() + if err != nil { + return fmt.Errorf("loading systemd socket activation listeners: %w", err) + } + activatedListener, err := selectSystemdListener(activatedListeners) + if err != nil { + for _, listener := range activatedListeners { + _ = listener.Close() + } + return err + } + if activatedListener != nil { + defer func() { + _ = activatedListener.Close() + }() + } + os.MkdirAll(r.BackendsPath, 0750) os.MkdirAll(r.ModelsPath, 0750) @@ -732,8 +749,13 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { // LAN, or VPN that's the historical "trusted network" deployment, but on // a public IP it makes every model, gallery install, settings change, and // admin endpoint reachable by anyone who can connect to the port. + listenAddress := r.Address + if activatedListener != nil { + listenAddress = activatedListener.Addr().String() + } + authConfigured := app.AuthDB() != nil || len(r.APIKeys) > 0 - if err := requireAuthOrTrustedBind(r.Address, authConfigured, r.AllowInsecurePublicBind); err != nil { + if err := requireAuthOrTrustedBind(listenAddress, authConfigured, r.AllowInsecurePublicBind); err != nil { return err } @@ -743,7 +765,11 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { return err } - xlog.Info("LocalAI is started and running", "address", r.Address) + if activatedListener != nil { + appHTTP.Listener = activatedListener + xlog.Info("Using systemd socket activation listener", "address", listenAddress) + } + xlog.Info("LocalAI is started and running", "address", listenAddress) // Start P2P if token was provided via CLI/env or loaded from runtime_settings.json if token != "" || app.ApplicationConfig().P2PToken != "" { @@ -762,11 +788,11 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { // backends like PostgreSQL need to call the embeddings API during // collection initialization. go func() { - waitForServerReady(r.Address, app.ApplicationConfig().Context) + waitForServerReady(listenAddress, app.ApplicationConfig().Context) app.StartAgentPool() }() - return appHTTP.Start(r.Address) + return appHTTP.Start(listenAddress) } // waitForServerReady polls the given address until the HTTP server is diff --git a/core/cli/run_socket_activation.go b/core/cli/run_socket_activation.go new file mode 100644 index 000000000..b809e3a48 --- /dev/null +++ b/core/cli/run_socket_activation.go @@ -0,0 +1,17 @@ +package cli + +import ( + "fmt" + "net" +) + +func selectSystemdListener(listeners []net.Listener) (net.Listener, error) { + switch len(listeners) { + case 0: + return nil, nil + case 1: + return listeners[0], nil + default: + return nil, fmt.Errorf("systemd socket activation requires exactly one stream listener, got %d", len(listeners)) + } +} diff --git a/core/cli/run_socket_activation_linux.go b/core/cli/run_socket_activation_linux.go new file mode 100644 index 000000000..35aade6cb --- /dev/null +++ b/core/cli/run_socket_activation_linux.go @@ -0,0 +1,71 @@ +//go:build linux + +package cli + +import ( + "fmt" + "net" + "os" + "strconv" +) + +const systemdListenFDStart = 3 + +func systemdActivatedListeners() ([]net.Listener, error) { + listenPID := os.Getenv("LISTEN_PID") + listenFDs := os.Getenv("LISTEN_FDS") + if listenPID == "" && listenFDs == "" { + return nil, nil + } + + defer func() { + for _, key := range []string{"LISTEN_PID", "LISTEN_FDS", "LISTEN_FDNAMES"} { + _ = os.Unsetenv(key) + } + }() + + pid, err := strconv.Atoi(listenPID) + if err != nil { + return nil, fmt.Errorf("invalid LISTEN_PID %q: %w", listenPID, err) + } + count, err := strconv.Atoi(listenFDs) + if err != nil || count < 0 { + return nil, fmt.Errorf("invalid LISTEN_FDS %q", listenFDs) + } + if pid != os.Getpid() || count == 0 { + return nil, nil + } + + return listenersFromSystemdFDs(systemdListenFDStart, count) +} + +func listenersFromSystemdFDs(start, count int) (_ []net.Listener, err error) { + listeners := make([]net.Listener, 0, count) + defer func() { + if err != nil { + for _, listener := range listeners { + _ = listener.Close() + } + } + }() + + for offset := range count { + fd := uintptr(start + offset) + file := os.NewFile(fd, fmt.Sprintf("LISTEN_FD_%d", fd)) + if file == nil { + return nil, fmt.Errorf("opening systemd listener file descriptor %d", fd) + } + listener, listenerErr := net.FileListener(file) + closeErr := file.Close() + if listenerErr != nil { + return nil, fmt.Errorf("using systemd file descriptor %d as a stream listener: %w", fd, listenerErr) + } + if closeErr != nil { + _ = listener.Close() + return nil, fmt.Errorf("closing inherited systemd file descriptor %d: %w", fd, closeErr) + } + listeners = append(listeners, listener) + } + + return listeners, nil +} diff --git a/core/cli/run_socket_activation_other.go b/core/cli/run_socket_activation_other.go new file mode 100644 index 000000000..1bbbf1469 --- /dev/null +++ b/core/cli/run_socket_activation_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package cli + +import "net" + +func systemdActivatedListeners() ([]net.Listener, error) { + return nil, nil +} diff --git a/core/cli/run_socket_activation_test.go b/core/cli/run_socket_activation_test.go new file mode 100644 index 000000000..09f973596 --- /dev/null +++ b/core/cli/run_socket_activation_test.go @@ -0,0 +1,101 @@ +//go:build linux + +package cli + +import ( + "net" + "os" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("selectSystemdListener", func() { + It("keeps normal address binding when systemd passes no listener", func() { + listener, err := selectSystemdListener(nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(listener).To(BeNil()) + }) + + It("uses the single stream listener passed by systemd", func() { + inherited, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(inherited.Close) + + listener, err := selectSystemdListener([]net.Listener{inherited}) + + Expect(err).NotTo(HaveOccurred()) + Expect(listener).To(BeIdenticalTo(inherited)) + }) + + It("rejects ambiguous activation with multiple stream listeners", func() { + first, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(first.Close) + second, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(second.Close) + + listener, err := selectSystemdListener([]net.Listener{first, second}) + + Expect(err).To(MatchError(ContainSubstring("exactly one"))) + Expect(listener).To(BeNil()) + }) +}) + +var _ = Describe("systemdActivatedListeners", func() { + It("turns an inherited TCP file descriptor into a working listener", func() { + original, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + file, err := original.(*net.TCPListener).File() + Expect(err).NotTo(HaveOccurred()) + Expect(original.Close()).To(Succeed()) + + listeners, err := listenersFromSystemdFDs(int(file.Fd()), 1) + Expect(err).NotTo(HaveOccurred()) + Expect(listeners).To(HaveLen(1)) + DeferCleanup(listeners[0].Close) + + client, err := net.Dial("tcp", listeners[0].Addr().String()) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(client.Close) + server, err := listeners[0].Accept() + Expect(err).NotTo(HaveOccurred()) + Expect(server.Close()).To(Succeed()) + }) + + It("ignores descriptors intended for another process and clears the activation environment", func() { + Expect(os.Setenv("LISTEN_PID", strconv.Itoa(os.Getpid()+1))).To(Succeed()) + Expect(os.Setenv("LISTEN_FDS", "1")).To(Succeed()) + Expect(os.Setenv("LISTEN_FDNAMES", "localai-http")).To(Succeed()) + DeferCleanup(func() { + _ = os.Unsetenv("LISTEN_PID") + _ = os.Unsetenv("LISTEN_FDS") + _ = os.Unsetenv("LISTEN_FDNAMES") + }) + + listeners, err := systemdActivatedListeners() + + Expect(err).NotTo(HaveOccurred()) + Expect(listeners).To(BeEmpty()) + Expect(os.Getenv("LISTEN_PID")).To(BeEmpty()) + Expect(os.Getenv("LISTEN_FDS")).To(BeEmpty()) + Expect(os.Getenv("LISTEN_FDNAMES")).To(BeEmpty()) + }) + + It("reports malformed activation metadata instead of silently binding another socket", func() { + Expect(os.Setenv("LISTEN_PID", strconv.Itoa(os.Getpid()))).To(Succeed()) + Expect(os.Setenv("LISTEN_FDS", "not-a-number")).To(Succeed()) + DeferCleanup(func() { + _ = os.Unsetenv("LISTEN_PID") + _ = os.Unsetenv("LISTEN_FDS") + }) + + listeners, err := systemdActivatedListeners() + + Expect(err).To(MatchError(ContainSubstring("LISTEN_FDS"))) + Expect(listeners).To(BeNil()) + }) +}) diff --git a/docs/content/getting-started/linux.md b/docs/content/getting-started/linux.md index 51e26057e..af4389328 100644 --- a/docs/content/getting-started/linux.md +++ b/docs/content/getting-started/linux.md @@ -59,6 +59,58 @@ After installation, you can: - Configure models in the models directory - Customize settings via environment variables or config files +## Start LocalAI on demand with systemd + +LocalAI accepts a single TCP listener passed through the systemd socket +activation protocol. This lets systemd listen on the public port and start +LocalAI only when the first client connects. + +Create `/etc/systemd/system/local-ai.socket`: + +```ini +[Unit] +Description=LocalAI API socket + +[Socket] +ListenStream=8080 +NoDelay=true + +[Install] +WantedBy=sockets.target +``` + +Create the matching `/etc/systemd/system/local-ai.service`: + +```ini +[Unit] +Description=LocalAI + +[Service] +Type=simple +User=localai +Group=localai +ExecStart=/usr/local/bin/local-ai run +WorkingDirectory=/var/lib/local-ai +``` + +Adjust the user, binary path, working directory, and model configuration for +your installation. Then enable the socket, not the service: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now local-ai.socket +``` + +The first connection to port 8080 starts `local-ai.service`; systemd holds that +connection until LocalAI is ready to accept it. `LOCALAI_ADDRESS` and +`--address` are ignored while an inherited listener is present. LocalAI +rejects activation with multiple stream listeners so it cannot silently choose +the wrong endpoint. + +For a Podman-managed container, configure Podman to preserve and pass the +systemd socket file descriptor into the container. The LocalAI process inside +the container consumes the same activation protocol. + ## Next Steps - [Try it out with examples](/basics/try/)