diff --git a/Dockerfile b/Dockerfile index 7e1e2f4c3..4ca9b3279 100644 --- a/Dockerfile +++ b/Dockerfile @@ -393,7 +393,12 @@ RUN go install github.com/mikefarah/yq/v4@latest # If you cannot find a more suitable place for an addition, this layer is a suitable place for it. FROM requirements-drivers -ENV HEALTHCHECK_ENDPOINT=http://localhost:8080/readyz +# Optional override for the HEALTHCHECK target. Left empty so healthcheck.sh +# derives the endpoint from the mode the container is actually running — the +# same image runs `local-ai run` (HTTP on 8080) and `local-ai worker` (HTTP on +# the gRPC base port minus one), and a hardcoded default marked every worker +# permanently unhealthy (#10987). Set it to pin an explicit URL. +ENV HEALTHCHECK_ENDPOINT="" ARG CUDA_MAJOR_VERSION=12 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility @@ -403,6 +408,7 @@ ENV NVIDIA_VISIBLE_DEVICES=all WORKDIR / COPY ./entrypoint.sh . +COPY ./scripts/build/healthcheck.sh . # Copy the binary COPY --from=builder /build/local-ai ./ @@ -413,9 +419,22 @@ RUN --mount=from=builder,src=/build/,dst=/mnt/build \ # Make sure the models directory exists RUN mkdir -p /models /backends /data -# Define the health check command -HEALTHCHECK --interval=1m --timeout=10m --retries=10 \ - CMD curl -f ${HEALTHCHECK_ENDPOINT} || exit 1 +# Define the health check command. +# +# --start-period is the knob for slow starts, not --timeout/--retries. Since +# #10949 a frontend's startup preload materializes HuggingFace artifacts before +# the HTTP server binds (31 GB observed on a live cluster), so a healthy replica +# can legitimately fail probes for a long time. Failures inside the start period +# leave the container `starting` instead of burning retries, and the period ends +# early on the first success — so a generous value costs a fast-starting +# container nothing. A process that actually died is handled by the restart +# policy, not by health. +# +# --timeout is a per-probe deadline: 10m meant a wedged probe could hang for ten +# minutes and stretch detection without bound. A localhost curl that has not +# answered in 10s is itself the fault being detected. +HEALTHCHECK --start-period=60m --interval=1m --timeout=10s --retries=3 \ + CMD /healthcheck.sh VOLUME /models /backends /configuration /data EXPOSE 8080 diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index 3b72823f2..0fc5ac634 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -46,17 +46,26 @@ const ( // It provides PUT/GET/POST endpoints for uploading, downloading, and allocating temp files, // as well as backend log REST and WebSocket endpoints when logStore is non-nil. // Auth is via Bearer token (registration token), using constant-time comparison. -func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, logStore ...*model.BackendLogStore) (*http.Server, error) { +// A nil readiness fails open, keeping /readyz's historical always-200 answer. +func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) { listener, err := net.Listen("tcp", addr) if err != nil { return nil, fmt.Errorf("listen %s: %w", addr, err) } - return StartFileTransferServerWithListener(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, logStore...) + return StartFileTransferServerWithReadiness(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, logStore...) } // StartFileTransferServerWithListener starts the server on an existing listener. // This avoids the TOCTOU race of closing a listener and re-binding to the same port. func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, logStore ...*model.BackendLogStore) (*http.Server, error) { + return StartFileTransferServerWithReadiness(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, nil, logStore...) +} + +// StartFileTransferServerWithReadiness is StartFileTransferServerWithListener +// plus a readiness gate for the /readyz probe. A nil readiness fails open, so +// the probe keeps its historical always-200 behaviour for callers that have no +// meaningful readiness signal to report. +func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) { if err := os.MkdirAll(stagingDir, 0750); err != nil { return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err) } @@ -131,18 +140,30 @@ func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir // Liveness/readiness probes — unauthenticated so container orchestrators // (Docker HEALTHCHECK, k8s probes) can hit them without the bearer token. - // Reaching this point means the listener is bound and the mux is serving. - healthHandler := func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - return + probe := func(check func() error) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + if err := check(); err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte("not ready: " + err.Error())) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) } - mux.HandleFunc("/readyz", healthHandler) - mux.HandleFunc("/healthz", healthHandler) + + // Liveness: reaching this point means the listener is bound and the mux is + // serving. Deliberately independent of readiness — a worker whose NATS link + // is momentarily down must not be restarted, or a NATS blip becomes a + // cluster-wide restart storm. + mux.HandleFunc("/healthz", probe(func() error { return nil })) + // Readiness: "can this worker actually accept work?" See WorkerReadiness. + mux.HandleFunc("/readyz", probe(readiness.Check)) addr := lis.Addr().String() server := &http.Server{ diff --git a/core/services/nodes/worker_readiness.go b/core/services/nodes/worker_readiness.go new file mode 100644 index 000000000..89530abc8 --- /dev/null +++ b/core/services/nodes/worker_readiness.go @@ -0,0 +1,75 @@ +package nodes + +import ( + "errors" + "sync/atomic" +) + +// WorkerReadiness is the gate behind a worker's /readyz probe. +// +// It exists because the worker's HTTP file-transfer server is started before +// the worker has connected to NATS, and must keep serving after NATS drops. +// The probe is therefore installed after the fact rather than passed as a +// value, and must be safe to read from HTTP handler goroutines while the +// startup goroutine is still installing it. +type WorkerReadiness struct { + probe atomic.Pointer[func() error] +} + +// Set installs (or replaces) the readiness probe. +func (r *WorkerReadiness) Set(fn func() error) { + if r == nil { + return + } + r.probe.Store(&fn) +} + +// Check reports whether the worker can accept work. A nil receiver, or one with +// no probe installed, fails open: callers that never wire readiness (the +// frontend's own file-transfer server, tests, embedders) keep the historical +// always-ready behaviour rather than being wedged out of rotation forever. +func (r *WorkerReadiness) Check() error { + if r == nil { + return nil + } + fn := r.probe.Load() + if fn == nil || *fn == nil { + return nil + } + return (*fn)() +} + +// natsConn is the slice of *messaging.Client the readiness probe needs. Kept +// as a local interface so this package does not import messaging (which would +// be an import cycle) and so tests can supply a fake. +type natsConn interface { + IsConnected() bool +} + +// ErrNATSDisconnected is reported by NATSReadiness when the worker has lost its +// NATS connection. +var ErrNATSDisconnected = errors.New("NATS connection is down: worker cannot receive work") + +// NATSReadiness builds the worker's readiness probe. +// +// A worker's real health is not "a port is open" — that is precisely the +// failure mode of issue #10987, where a process that serves nothing still +// answered 200. All of a worker's actual work (backend install/start/stop +// events, inference dispatch, file-staging notifications) arrives over NATS, so +// a worker with a dead NATS link is up and useless. Registration is already +// implied by the probe being reachable at all: the file-transfer server is only +// started after the worker has successfully registered with the frontend. +// +// This is deliberately something the controller cannot already see. The node +// registry's status/last_heartbeat is fed by an HTTP heartbeat to the frontend, +// a completely different network path — a worker can keep heartbeating happily +// while its NATS connection is dead, and look healthy in the registry. The +// local probe closes that gap. +func NATSReadiness(conn natsConn) func() error { + return func() error { + if conn == nil || !conn.IsConnected() { + return ErrNATSDisconnected + } + return nil + } +} diff --git a/core/services/nodes/worker_readiness_test.go b/core/services/nodes/worker_readiness_test.go new file mode 100644 index 000000000..6771291ce --- /dev/null +++ b/core/services/nodes/worker_readiness_test.go @@ -0,0 +1,101 @@ +package nodes + +import ( + "errors" + "net" + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// fakeConn stands in for *messaging.Client, which cannot be constructed without +// a live NATS server. Only IsConnected() is consulted by the readiness probe. +type fakeConn struct{ connected bool } + +func (f *fakeConn) IsConnected() bool { return f.connected } + +var _ = Describe("WorkerReadiness", func() { + Describe("the gate itself", func() { + It("reports ready when no probe has been installed yet", func() { + // Fail open: an embedder (or the frontend) that never wires a probe + // keeps the historical always-ready behaviour. + r := &WorkerReadiness{} + Expect(r.Check()).To(Succeed()) + }) + + It("surfaces the installed probe's error", func() { + r := &WorkerReadiness{} + r.Set(func() error { return errors.New("boom") }) + Expect(r.Check()).To(MatchError(ContainSubstring("boom"))) + }) + + It("lets a later Set replace an earlier probe", func() { + r := &WorkerReadiness{} + r.Set(func() error { return errors.New("boom") }) + r.Set(func() error { return nil }) + Expect(r.Check()).To(Succeed()) + }) + }) + + Describe("NATSReadiness", func() { + It("reports ready while the NATS connection is up", func() { + Expect(NATSReadiness(&fakeConn{connected: true})()).To(Succeed()) + }) + + It("reports not-ready once the NATS connection drops", func() { + // This is the failure mode issue #10987 is about: the process is + // up and the port is bound, but the worker can receive no work. + err := NATSReadiness(&fakeConn{connected: false})() + Expect(err).To(MatchError(ContainSubstring("NATS"))) + }) + }) + + Describe("the file transfer server probes", func() { + var ( + srv *http.Server + baseURL string + ready *WorkerReadiness + ) + + BeforeEach(func() { + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + ready = &WorkerReadiness{} + srv, err = StartFileTransferServerWithReadiness( + lis, GinkgoT().TempDir(), GinkgoT().TempDir(), GinkgoT().TempDir(), + "tok", 1024, ready, nil, + ) + Expect(err).ToNot(HaveOccurred()) + baseURL = "http://" + lis.Addr().String() + DeferCleanup(func() { ShutdownFileTransferServer(srv) }) + }) + + get := func(path string) int { + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(baseURL + path) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode + } + + It("serves /readyz 200 while the probe reports ready", func() { + ready.Set(func() error { return nil }) + Expect(get("/readyz")).To(Equal(http.StatusOK)) + }) + + It("serves /readyz 503 once the probe reports not-ready", func() { + ready.Set(func() error { return errors.New("NATS disconnected") }) + Expect(get("/readyz")).To(Equal(http.StatusServiceUnavailable)) + }) + + It("keeps /healthz at 200 even when readiness fails", func() { + // Liveness is deliberately independent of readiness: a worker whose + // NATS link is briefly down must not be killed and restarted, or a + // NATS outage turns into a restart storm across every worker. + ready.Set(func() error { return errors.New("NATS disconnected") }) + Expect(get("/healthz")).To(Equal(http.StatusOK)) + }) + }) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 04376242a..2c48c14ea 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -149,7 +149,12 @@ func Run(ctx *cliContext.Context, cfg *Config) error { httpAddr := cfg.resolveHTTPAddr() stagingDir := filepath.Join(cfg.ModelsPath, "..", "staging") dataDir := filepath.Join(cfg.ModelsPath, "..", "data") - httpServer, err := nodes.StartFileTransferServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, ml.BackendLogs()) + // The readiness gate is created here but only armed once NATS is up, below. + // Until then /readyz reports ready, which is correct: reaching this line + // means the worker has already registered with the frontend, so it is + // mid-startup rather than broken. + readiness := &nodes.WorkerReadiness{} + httpServer, err := nodes.StartFileTransferServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, readiness, ml.BackendLogs()) if err != nil { return fmt.Errorf("starting HTTP file transfer server: %w", err) } @@ -163,6 +168,11 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } defer natsClient.Close() + // Arm the readiness gate now that the worker can actually receive work. + // From here /readyz tracks the live NATS link, so a worker that is up but + // cut off from the bus reports 503 instead of a meaningless 200 (#10987). + readiness.Set(nodes.NATSReadiness(natsClient)) + // Start heartbeat goroutine (after NATS is connected so IsConnected check works) go func() { ticker := time.NewTicker(heartbeatInterval) diff --git a/docker-compose.distributed.yaml b/docker-compose.distributed.yaml index 81397e4ac..9971b57c3 100644 --- a/docker-compose.distributed.yaml +++ b/docker-compose.distributed.yaml @@ -101,12 +101,12 @@ services: - BASE_IMAGE=ubuntu:24.04 command: - worker - # The image's default HEALTHCHECK targets the server's /readyz on 8080. - # Workers don't run the OpenAI API server — their HTTP file transfer - # server runs on the gRPC base port - 1 (50050 here) and exposes /readyz. - # Override the env var so the inherited HEALTHCHECK probes the right port. + # No HEALTHCHECK_ENDPOINT override is needed: the image's healthcheck + # detects worker mode and derives the port from LOCALAI_SERVE_ADDR below + # (gRPC base port - 1 = 50050). The worker's /readyz reports 503 while its + # NATS connection is down, so `unhealthy` here means the worker genuinely + # cannot receive work. environment: - HEALTHCHECK_ENDPOINT: "http://localhost:50050/readyz" LOCALAI_SERVE_ADDR: "0.0.0.0:50051" LOCALAI_ADVERTISE_ADDR: "worker-1:50051" LOCALAI_ADVERTISE_HTTP_ADDR: "worker-1:50050" @@ -200,10 +200,9 @@ services: - | apt-get update -qq && apt-get install -y -qq docker.io >/dev/null 2>&1 exec /entrypoint.sh agent-worker - # The agent worker is NATS-only — no HTTP server to probe. Disable the - # image's inherited HEALTHCHECK so the container doesn't show unhealthy. - healthcheck: - disable: true + # The agent worker is NATS-only — no HTTP server to probe. The image's + # healthcheck detects that mode and reports healthy rather than probing a + # port that will never bind, so no override is needed here. environment: LOCALAI_NATS_URL: "nats://nats:4222" LOCALAI_REGISTER_TO: "http://localai:8080" diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index e97283346..bd5441672 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -232,6 +232,19 @@ local-ai worker \ **HTTP file transfer:** Each worker also runs a small HTTP server for file transfer (model files, configs). By default it listens on the gRPC base port - 1 (e.g., if gRPC base is 50051, HTTP is on 50050). gRPC ports grow upward from the base port as additional models are loaded. Set `--advertise-http-addr` if the auto-detected address is not routable from the frontend. {{% /notice %}} +### Worker Health Probes + +The worker's HTTP server (base port - 1, default 50050) exposes two unauthenticated probes: + +| Endpoint | Meaning | +|----------|---------| +| `/healthz` | **Liveness.** 200 whenever the process is up and serving. Deliberately independent of readiness, so a brief NATS outage does not trigger a restart storm across every worker. | +| `/readyz` | **Readiness.** 200 only when the worker is registered *and* its NATS connection is live; 503 otherwise. | + +`/readyz` reports something the frontend cannot see on its own. The node registry's `status` and `last_heartbeat` are driven by an HTTP heartbeat to the frontend, which is a different network path from NATS — a worker can keep heartbeating while its NATS link is dead, and so appear `healthy` in the registry while being unable to receive any work. The local probe closes that gap. + +The container image's `HEALTHCHECK` detects worker mode and probes this endpoint automatically; no `HEALTHCHECK_ENDPOINT` override is needed. Set `HEALTHCHECK_ENDPOINT` only to pin an explicit URL. + ### Worker Address Configuration The simplest way to configure a worker's network address is with a single variable: diff --git a/docs/content/getting-started/containers.md b/docs/content/getting-started/containers.md index ad2d81fa8..eb3d80f16 100644 --- a/docs/content/getting-started/containers.md +++ b/docs/content/getting-started/containers.md @@ -111,9 +111,14 @@ services: # For CUDA 13, use: localai/localai:latest-gpu-nvidia-cuda-13 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] + # start_period, not timeout, is the knob for a slow first boot: startup + # preload can download tens of GB before the API binds, and failures + # inside the start period leave the container `starting` rather than + # marking it unhealthy. timeout is a per-probe deadline. + start_period: 60m interval: 1m - timeout: 20m - retries: 5 + timeout: 10s + retries: 3 ports: - 8080:8080 environment: @@ -151,9 +156,14 @@ services: # For CUDA 13, use: localai/localai:latest-gpu-nvidia-cuda-13 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] + # start_period, not timeout, is the knob for a slow first boot: startup + # preload can download tens of GB before the API binds, and failures + # inside the start period leave the container `starting` rather than + # marking it unhealthy. timeout is a per-probe deadline. + start_period: 60m interval: 1m - timeout: 20m - retries: 5 + timeout: 10s + retries: 3 ports: - 8080:8080 environment: diff --git a/scripts/build/healthcheck.sh b/scripts/build/healthcheck.sh new file mode 100755 index 000000000..eff574ded --- /dev/null +++ b/scripts/build/healthcheck.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# Docker HEALTHCHECK command for the LocalAI image. +# +# The image is a single artifact that runs several different processes, and they +# do not all serve HTTP on the same port — or at all. A hardcoded +# `curl -f http://localhost:8080/readyz` therefore reported every `local-ai +# worker` container as permanently `unhealthy` (issue #10987), which is worse +# than no healthcheck: a genuinely broken worker and a perfectly good one both +# read `unhealthy`, so the signal carries no information. +# +# The endpoint is derived from the mode the container is actually running plus +# the same env vars that configure the bind address, so a frontend moved off +# 8080 and a worker on a non-default base port are both probed correctly. +# +# Precedence: +# 1. HEALTHCHECK_ENDPOINT, if set — the documented escape hatch, and what +# docker-compose.distributed.yaml has been shipping as a workaround. +# 2. Derived from the running mode. +# 3. The frontend endpoint, when the mode cannot be determined. +# +# Ports are read from environment variables only, which is how containers are +# configured in practice (compose/k8s set LOCALAI_ADDRESS, LOCALAI_SERVE_ADDR, +# ...). If you instead pass the bind address as a CLI flag, set +# HEALTHCHECK_ENDPOINT to match. +set -u + +# Detect the arguments local-ai was started with. PID 1 is the usual case +# (entrypoint.sh exec's local-ai, so it inherits PID 1), but `init: true` puts +# docker-init there instead, so fall back to scanning /proc. Reading /proc +# directly avoids depending on ps/pgrep being installed in every image variant. +detect_argv() { + if [ -n "${LOCALAI_HEALTHCHECK_ARGV:-}" ]; then + printf '%s' "$LOCALAI_HEALTHCHECK_ARGV" + return + fi + + local cmdline proc pid d + # LOCALAI_HEALTHCHECK_PROC exists so the regression test can point this at a + # fixture tree; nothing in the image sets it. + local procfs="${LOCALAI_HEALTHCHECK_PROC:-/proc}" + # PID 1 first: entrypoint.sh exec's local-ai, so it normally *is* PID 1. + # Only when something else holds PID 1 (`init: true` puts docker-init there) + # do we scan, lowest PID first so the answer is deterministic if a container + # somehow has more than one local-ai process. The healthcheck's own shell is + # skipped: its argv mentions the script path, not a mode. + # Sort on the PID itself rather than the whole path, so 7 comes before 64. + for pid in 1 $(for d in "$procfs"/[0-9]*; do basename "$d"; done | sort -n); do + proc="$procfs/$pid" + [ -r "$proc/cmdline" ] || continue + cmdline=$(tr '\0' ' ' < "$proc/cmdline" 2>/dev/null) || continue + case "$cmdline" in + *healthcheck.sh*) continue ;; + *local-ai*) + printf '%s' "$cmdline" + return + ;; + esac + done +} + +# Extract the port from a bind address. Accepts ":8080", "0.0.0.0:8080" and +# "host:8080"; prints nothing when there is no port to find. +port_of() { + case "$1" in + *:*) printf '%s' "${1##*:}" ;; + esac +} + +# The mode is the first non-flag word after the local-ai binary. +# +# `run` is kong's default command, and it is declared `default:"withargs"` — so +# `local-ai gemma-4 whisper` is the *frontend* with two model arguments, not a +# command called "gemma-4". Unrecognised words must therefore fall through to +# the frontend; treating them as an unknown mode would silently stop probing the +# single most common invocation in the docs. +detect_mode() { + local seen_binary=0 word + for word in $1; do + if [ "$seen_binary" = 0 ]; then + case "$word" in + *local-ai) seen_binary=1 ;; + esac + continue + fi + case "$word" in + -*) continue ;; + *) printf '%s' "$word"; return ;; + esac + done + printf 'run' +} + +endpoint="${HEALTHCHECK_ENDPOINT:-}" + +if [ -z "$endpoint" ]; then + mode=$(detect_mode "$(detect_argv)") + case "$mode" in + worker) + # The worker's file-transfer server (which also serves /readyz and + # /healthz) binds LOCALAI_HTTP_ADDR when set, otherwise the gRPC + # base port minus one. See Config.resolveHTTPAddr. + port=$(port_of "${LOCALAI_HTTP_ADDR:-}") + if [ -z "$port" ]; then + base=$(port_of "${LOCALAI_SERVE_ADDR:-}") + port=$(( ${base:-50051} - 1 )) + fi + endpoint="http://localhost:${port}/readyz" + ;; + agent-worker|p2p-worker|chat|models|backends|tts|sound-generation|transcript|util|agent|mcp-server|completion) + # Modes with no HTTP surface of their own — agent-worker and + # p2p-worker are message-bus only, and the rest are one-shot + # commands that exit on their own. Claiming `unhealthy` for a + # process that was never going to bind a port is the same false + # signal as #10987, one mode over. + exit 0 + ;; + *) + # run / federated / explorer, and anything unrecognised: kong's + # default command is `run`, so a bare `local-ai`, `local-ai --flag` + # and `local-ai ` are all frontends on the API port. + port=$(port_of "${LOCALAI_ADDRESS:-${ADDRESS:-}}") + endpoint="http://localhost:${port:-8080}/readyz" + ;; + esac +fi + +# Docker only distinguishes 0 from non-zero; normalise curl's exit codes (22 for +# the 503 a still-preloading frontend returns, 7 for connection refused) to 1 so +# the status is unambiguous in `docker inspect`. +curl -fsS -m 10 "$endpoint" >/dev/null 2>&1 || exit 1 +exit 0 diff --git a/scripts/build/healthcheck_test.sh b/scripts/build/healthcheck_test.sh new file mode 100644 index 000000000..86d2ff098 --- /dev/null +++ b/scripts/build/healthcheck_test.sh @@ -0,0 +1,175 @@ +#!/bin/bash +# Regression test for scripts/build/healthcheck.sh, the image's HEALTHCHECK +# command. +# +# The bug this guards (issue #10987): the Dockerfile hardcoded +# http://localhost:8080/readyz, but the same image also runs `local-ai worker`, +# which serves HTTP on the gRPC base port minus one (50050 by default) and never +# binds 8080. Every worker container was therefore permanently `unhealthy`, +# which made the health signal useless — a genuinely broken worker looked +# exactly like a working one. +# +# The same hardcoding also broke any frontend moved off port 8080 with +# LOCALAI_ADDRESS, which is why the derivation is tested for both modes. +# +# The script probes no network here: curl is stubbed with a script that records +# the URL it was asked for, so these assertions are about URL derivation and +# exit status only. Needs only bash. +set -euo pipefail + +CURDIR=$(dirname "$(realpath "$0")") +SCRIPT="$CURDIR/healthcheck.sh" + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# Stub curl: record the last URL argument, succeed or fail per STUB_CURL_EXIT. +mkdir -p "$WORK/bin" +cat > "$WORK/bin/curl" <<'EOF' +#!/bin/bash +for arg in "$@"; do + case "$arg" in + http*) echo "$arg" > "$CURL_URL_FILE" ;; + esac +done +exit "${STUB_CURL_EXIT:-0}" +EOF +chmod +x "$WORK/bin/curl" + +export CURL_URL_FILE="$WORK/url" +PATH="$WORK/bin:$PATH" +export PATH + +FAILED=0 + +# run_hc [env assignments...] +run_hc() { + local want_exit="$1"; shift + local argv="$1"; shift + : > "$CURL_URL_FILE" + local got_exit=0 + env "$@" LOCALAI_HEALTHCHECK_ARGV="$argv" bash "$SCRIPT" >/dev/null 2>&1 || got_exit=$? + if [ "$got_exit" != "$want_exit" ]; then + echo "FAIL: argv='$argv' env='$*' expected exit $want_exit, got $got_exit" + FAILED=1 + fi +} + +expect_url() { + local want="$1" + local got + got=$(cat "$CURL_URL_FILE" 2>/dev/null || true) + if [ "$got" != "$want" ]; then + echo "FAIL: expected probe of '$want', got '$got'" + FAILED=1 + fi +} + +echo "== frontend defaults to 8080" +run_hc 0 "local-ai run" +expect_url "http://localhost:8080/readyz" + +echo "== frontend with no subcommand still probes the API port" +# `run` is the kong default command, so a bare `local-ai` is a frontend too. +run_hc 0 "local-ai" +expect_url "http://localhost:8080/readyz" + +echo "== a bare model list is the frontend, not an unknown mode" +# `run` is declared default:"withargs", so `local-ai gemma-4 whisper` is the +# frontend with two model arguments. Classifying "gemma-4" as an unknown mode +# would silently stop probing the most common invocation in the docs. +run_hc 0 "local-ai gemma-4 whisper-1" +expect_url "http://localhost:8080/readyz" + +echo "== leading flags do not hide the mode" +run_hc 0 "local-ai --debug worker" +expect_url "http://localhost:50050/readyz" + +echo "== frontend honours LOCALAI_ADDRESS" +run_hc 0 "local-ai run" LOCALAI_ADDRESS=":9090" +expect_url "http://localhost:9090/readyz" + +echo "== frontend honours the legacy ADDRESS alias" +run_hc 0 "local-ai run" ADDRESS="0.0.0.0:7070" +expect_url "http://localhost:7070/readyz" + +echo "== worker defaults to the file-transfer port (base 50051 - 1)" +run_hc 0 "local-ai worker" +expect_url "http://localhost:50050/readyz" + +echo "== worker derives the port from LOCALAI_SERVE_ADDR" +run_hc 0 "local-ai worker" LOCALAI_SERVE_ADDR="0.0.0.0:60000" +expect_url "http://localhost:59999/readyz" + +echo "== worker honours an explicit LOCALAI_HTTP_ADDR" +run_hc 0 "local-ai worker" LOCALAI_HTTP_ADDR="0.0.0.0:18080" +expect_url "http://localhost:18080/readyz" + +echo "== an explicit HEALTHCHECK_ENDPOINT overrides derivation" +# docker-compose.distributed.yaml has shipped this override as the workaround, +# so it must keep winning. +run_hc 0 "local-ai worker" HEALTHCHECK_ENDPOINT="http://localhost:50050/readyz" +expect_url "http://localhost:50050/readyz" + +echo "== a failing probe propagates a non-zero exit" +run_hc 1 "local-ai run" STUB_CURL_EXIT=1 + +echo "== a 503 from a still-preloading frontend is unhealthy, not a crash" +# curl -f exits 22 on 503; the healthcheck must report failure (Docker only +# distinguishes 0 from non-zero) rather than masking it. +run_hc 1 "local-ai run" STUB_CURL_EXIT=22 + +echo "== modes with no HTTP surface report healthy rather than false-unhealthy" +# agent-worker is NATS-only. Reporting `unhealthy` forever for a process that +# was never going to bind a port is the same bug as #10987, one mode over. +run_hc 0 "local-ai agent-worker" +expect_url "" + +echo "== an argv with no local-ai in it falls back to the frontend endpoint" +# e.g. `init: true` puts docker-init at PID 1. Guessing the frontend is the +# safer default than skipping the probe entirely. +run_hc 0 "/sbin/docker-init" +expect_url "http://localhost:8080/readyz" + +echo "== detects the mode from /proc when no argv override is given" +# The path that actually runs in the image. LOCALAI_HEALTHCHECK_ARGV is unset +# here, so this exercises detect_argv against a fixture /proc tree. +make_proc() { + local root="$1"; shift + rm -rf "$root"; mkdir -p "$root" + while [ $# -gt 0 ]; do + mkdir -p "$root/$1" + printf '%s' "$2" | tr ' ' '\0' > "$root/$1/cmdline" + shift 2 + done +} + +probe_proc() { + local root="$1"; shift + : > "$CURL_URL_FILE" + env "$@" LOCALAI_HEALTHCHECK_PROC="$root" bash "$SCRIPT" >/dev/null 2>&1 || true +} + +# The normal container shape: local-ai exec'd by entrypoint.sh, so it is PID 1. +make_proc "$WORK/proc-worker" 1 "./local-ai worker" +probe_proc "$WORK/proc-worker" LOCALAI_SERVE_ADDR="0.0.0.0:50051" +expect_url "http://localhost:50050/readyz" + +make_proc "$WORK/proc-frontend" 1 "./local-ai run" +probe_proc "$WORK/proc-frontend" +expect_url "http://localhost:8080/readyz" + +# `init: true`: docker-init holds PID 1, so the scan has to find local-ai +# further down, and must prefer the lowest PID deterministically. +make_proc "$WORK/proc-init" \ + 1 "/sbin/docker-init --" \ + 7 "./local-ai worker" \ + 64 "./local-ai run" +probe_proc "$WORK/proc-init" LOCALAI_SERVE_ADDR="0.0.0.0:50051" +expect_url "http://localhost:50050/readyz" + +if [ "$FAILED" != 0 ]; then + echo "FAILED" + exit 1 +fi +echo "OK"