// Package workerregistry provides a shared HTTP client for worker node // registration, heartbeating, draining, and deregistration against a // LocalAI frontend. Both the backend worker (WorkerCMD) and the agent // worker (AgentWorkerCMD) use this instead of duplicating the logic. package workerregistry import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "strings" "sync" "time" "github.com/mudler/xlog" "github.com/mudler/LocalAI/pkg/httpclient" ) // RegistrationClient talks to the frontend's /api/node/* endpoints. type RegistrationClient struct { FrontendURL string RegistrationToken string HTTPTimeout time.Duration // used for registration calls; defaults to 10s client *http.Client clientOnce sync.Once } // httpTimeout returns the configured timeout or a sensible default. func (c *RegistrationClient) httpTimeout() time.Duration { if c.HTTPTimeout > 0 { return c.HTTPTimeout } return 10 * time.Second } // httpClient returns the shared HTTP client, initializing it on first use. func (c *RegistrationClient) httpClient() *http.Client { c.clientOnce.Do(func() { c.client = httpclient.NewWithTimeout(c.httpTimeout()) }) return c.client } // baseURL returns FrontendURL with any trailing slash stripped. func (c *RegistrationClient) baseURL() string { return strings.TrimRight(c.FrontendURL, "/") } // setAuth adds an Authorization header when a token is configured. func (c *RegistrationClient) setAuth(req *http.Request) { if c.RegistrationToken != "" { req.Header.Set("Authorization", "Bearer "+c.RegistrationToken) } } // RegisterResponse is the JSON body returned by /api/node/register. type RegisterResponse struct { ID string `json:"id"` Status string `json:"status,omitempty"` // "pending" until an admin approves the node APIToken string `json:"api_token,omitempty"` // TunnelToken is this node's own credential for GET /api/cluster/connect. // The frontend mints a fresh one on every registration and keeps only its // hash, so this is the ONLY time the plaintext exists anywhere but in this // worker's memory: a worker that discards it cannot get it back without // registering again. TunnelToken string `json:"tunnel_token,omitempty"` // There are no nats_jwt / nats_user_seed fields. A frontend that predates // this release still sends them and this decodes fine: encoding/json // ignores a key with no field, so an old frontend talking to a new worker // is a no-op rather than a decode failure. } // RegisterFull sends a single registration request and returns the full // response (node ID, approval status, and optional API and tunnel tokens). // Re-registration is idempotent: the frontend preserves the node row and mints // a fresh TUNNEL token each call, so this doubles as the rotation call. It is // the only credential a registration mints; the per-node broker JWT it used to // carry went with the bus. func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]any) (*RegisterResponse, error) { jsonBody, _ := json.Marshal(body) url := c.baseURL() + "/api/node/register" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody)) if err != nil { return nil, fmt.Errorf("creating request: %w", err) } req.Header.Set("Content-Type", "application/json") c.setAuth(req) resp, err := c.httpClient().Do(req) if err != nil { return nil, fmt.Errorf("posting to %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, registrationStatusError(resp) } var result RegisterResponse if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, fmt.Errorf("decoding response: %w", err) } return &result, nil } // ErrRegistrationRejected marks a registration the frontend REFUSED, as opposed // to one it could not answer. // // Retrying a refusal cannot change it: the request is wrong, or this worker is // not allowed to make it. The one that matters in practice is a worker of this // release registering against a frontend that predates it, which answers // "address is required for backend workers" with 400, because a worker no // longer has an address to send. Without this the retry ladder spends four // minutes on a verdict the frontend reached instantly, and the operator watches // it before being told anything. // // 408 and 429 are deliberately NOT rejections. Both are the frontend asking for // the same request again later, which is exactly what a retry does. var ErrRegistrationRejected = errors.New("the frontend refused this registration") // maxRegistrationErrorBody bounds how much of a refusal's body is quoted back. // Enough for a message, not enough for an HTML error page to bury the log line // it is meant to explain. const maxRegistrationErrorBody = 512 // registrationStatusError turns a non-2xx response into an error that says WHY. // // The body is the point. The frontend explains its refusals there // ("address is required for backend workers", "invalid registration token"), // and discarding it left an operator with a bare status code: the one line that // would tell them which of several possible mistakes they made was read off the // socket and thrown away. func registrationStatusError(resp *http.Response) error { detail, err := io.ReadAll(io.LimitReader(resp.Body, maxRegistrationErrorBody)) if err != nil { xlog.Debug("Could not read the frontend's registration error body", "status", resp.StatusCode, "error", err) } msg := strings.Join(strings.Fields(string(detail)), " ") base := fmt.Sprintf("registration failed with status %d", resp.StatusCode) if msg != "" { base = fmt.Sprintf("%s: %s", base, msg) } if isRegistrationRejection(resp.StatusCode) { return fmt.Errorf("%s: %w", base, ErrRegistrationRejected) } return errors.New(base) } // isRegistrationRejection reports whether a status is a verdict rather than a // condition that may pass. func isRegistrationRejection(status int) bool { if status == http.StatusRequestTimeout || status == http.StatusTooManyRequests { return false } return status >= 400 && status < 500 } // RegisterWithRetry retries registration with exponential backoff. // // It drops every field of the response it does not name, the tunnel credential // among them. Callers that need one use RegisterFullWithRetry. // // The two broker-credential returns it used to carry are gone with the bus, and // so is the Register one-shot that existed only to carry them: it had no caller // left once nothing dialled a broker. func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken string, err error) { res, err := c.RegisterFullWithRetry(ctx, body, maxRetries) if err != nil { return "", "", err } return res.ID, res.APIToken, nil } // RegisterFullWithRetry retries registration with exponential backoff and // returns the whole response. func (c *RegistrationClient) RegisterFullWithRetry(ctx context.Context, body map[string]any, maxRetries int) (*RegisterResponse, error) { backoff := 2 * time.Second maxBackoff := 30 * time.Second var err error for attempt := 1; attempt <= maxRetries; attempt++ { var res *RegisterResponse res, err = c.RegisterFull(ctx, body) if err == nil { return res, nil } if errors.Is(err, ErrRegistrationRejected) { // A verdict, not an outage. Reported on the first attempt so the // reason the frontend gave is the first thing in the log rather // than the last, after the ladder. return nil, err } if attempt == maxRetries { return nil, fmt.Errorf("failed after %d attempts: %w", maxRetries, err) } xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) select { case <-ctx.Done(): return nil, ctx.Err() case <-time.After(backoff): } backoff = min(backoff*2, maxBackoff) } return nil, err } // Heartbeat sends a single heartbeat POST with the given body. func (c *RegistrationClient) Heartbeat(ctx context.Context, nodeID string, body map[string]any) error { jsonBody, _ := json.Marshal(body) url := c.baseURL() + "/api/node/" + nodeID + "/heartbeat" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody)) if err != nil { return fmt.Errorf("creating heartbeat request: %w", err) } req.Header.Set("Content-Type", "application/json") c.setAuth(req) resp, err := c.httpClient().Do(req) if err != nil { return err } defer resp.Body.Close() return nil } // HeartbeatLoop runs heartbeats at the given interval until ctx is cancelled. // bodyFn is called each tick to build the heartbeat payload (e.g. VRAM stats). func (c *RegistrationClient) HeartbeatLoop(ctx context.Context, nodeID string, interval time.Duration, bodyFn func() map[string]any) { ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: body := bodyFn() if err := c.Heartbeat(ctx, nodeID, body); err != nil { xlog.Warn("Heartbeat failed", "error", err) } } } } // Drain sets the node to draining status via POST /api/node/:id/drain. func (c *RegistrationClient) Drain(ctx context.Context, nodeID string) error { url := c.baseURL() + "/api/node/" + nodeID + "/drain" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { return fmt.Errorf("creating drain request: %w", err) } c.setAuth(req) resp, err := c.httpClient().Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("drain failed with status %d", resp.StatusCode) } return nil } // WaitForDrain polls GET /api/node/:id/models until all models report 0 // in-flight requests, or until timeout elapses. func (c *RegistrationClient) WaitForDrain(ctx context.Context, nodeID string, timeout time.Duration) { url := c.baseURL() + "/api/node/" + nodeID + "/models" deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { xlog.Warn("Failed to create drain poll request", "error", err) return } c.setAuth(req) resp, err := c.httpClient().Do(req) if err != nil { xlog.Warn("Drain poll failed, will retry", "error", err) select { case <-ctx.Done(): xlog.Warn("Drain wait cancelled") return case <-time.After(1 * time.Second): } continue } var models []struct { InFlight int `json:"in_flight"` } json.NewDecoder(resp.Body).Decode(&models) resp.Body.Close() total := 0 for _, m := range models { total += m.InFlight } if total == 0 { xlog.Info("All in-flight requests drained") return } xlog.Info("Waiting for in-flight requests", "count", total) select { case <-ctx.Done(): xlog.Warn("Drain wait cancelled") return case <-time.After(1 * time.Second): } } xlog.Warn("Drain timeout reached, proceeding with shutdown") } // Deregister marks the node as offline via POST /api/node/:id/deregister. // The node row is preserved in the database so re-registration restores // approval status. func (c *RegistrationClient) Deregister(ctx context.Context, nodeID string) error { url := c.baseURL() + "/api/node/" + nodeID + "/deregister" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) if err != nil { return fmt.Errorf("creating deregister request: %w", err) } c.setAuth(req) resp, err := c.httpClient().Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("deregistration failed with status %d", resp.StatusCode) } return nil } // GracefulDeregister performs drain -> wait -> deregister in sequence. // This is the standard shutdown sequence for backend workers. func (c *RegistrationClient) GracefulDeregister(nodeID string) { if c.FrontendURL == "" || nodeID == "" { return } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() if err := c.Drain(ctx, nodeID); err != nil { xlog.Warn("Failed to set drain status", "error", err) } else { c.WaitForDrain(ctx, nodeID, 30*time.Second) } if err := c.Deregister(ctx, nodeID); err != nil { xlog.Error("Failed to deregister", "error", err) } else { xlog.Info("Deregistered from frontend") } }