fix(e2e): wire proxy api_key_env lookup into the e2e in-process app

19f6a18b8 moved credential-env resolution behind an explicit lookup
(ApplicationConfig.ProxyAPIKeyEnvLookup / config.WithProxyAPIKeyEnvLookup),
wired only at the CLI boundary (core/cli/run.go). The e2e suite builds its
Application in-process without that option, so the failover prober could
never resolve a remote target's api_key_env, remote liveness never passed,
and "fails over ... and fails back" hung waiting for chain-remote to
recover. Pass config.WithProxyAPIKeyEnvLookup(os.Getenv) there too, same as
the CLI. worker/federated commands don't serve proxy/failover configs and
tests/e2e-ui never sets api_key_env, so neither needs the lookup.

Also make the misconfiguration itself easier to diagnose: the prober now
logs a one-time xlog.Warn per api_key_env when a remote target sets it but
no lookup is configured, instead of only surfacing it as a per-probe
"is unset" error indistinguishable from a genuinely empty env var.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/services/failover/prober.go

@@ -12,10 +12,12 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/grpc"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/xlog"
 )
 
 // ErrNotLoaded is what Inference returns for a local target whose backend is
@@ -35,6 +37,11 @@ type DefaultProber struct {
 	HTTP      *http.Client
 	Loaded    LoadedFunc
 	EnvLookup func(string) string
+
+	// unresolvableEnvWarned tracks which api_key_env names already got the
+	// "no lookup configured" warning below, so a chain with no working
+	// credential lookup does not re-log on every probe tick.
+	unresolvableEnvWarned sync.Map // map[string]struct{}
 }
 
 func NewProber(loaded LoadedFunc, envLookup func(string) string) *DefaultProber {
@@ -103,6 +110,11 @@ func PrepareTarget(cfg *config.ModelConfig) {
 }
 
 func (p *DefaultProber) authorize(req *http.Request, cfg config.ModelConfig) error {
+	if cfg.Proxy.APIKeyEnv != "" && p.EnvLookup == nil {
+		if _, warned := p.unresolvableEnvWarned.LoadOrStore(cfg.Proxy.APIKeyEnv, struct{}{}); !warned {
+			xlog.Warn("failover: remote target has api_key_env set but no credential lookup is configured; liveness and requests will fail authorization", "target", cfg.Name, "api_key_env", cfg.Proxy.APIKeyEnv)
+		}
+	}
 	key, err := cfg.Proxy.ResolveAPIKey(p.EnvLookup)
 	if err != nil || key == "" {
 		return err

core/services/failover/prober_test.go

@@ -1,23 +1,40 @@
 package failover
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/grpc"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/xlog"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	ggrpc "google.golang.org/grpc"
 )
 
+// captureXlog redirects the package-wide xlog logger to buf for the duration
+// of a test and restores the suite's default on cleanup. xlog exposes no
+// getter for the current logger, so this restores the same default the
+// entrypoint installs rather than the prior value (same pattern as
+// core/config/model_artifact_fallback_test.go).
+func captureXlog(buf *bytes.Buffer) {
+	handler := slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelWarn})
+	xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelWarn))
+	DeferCleanup(func() {
+		xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
+	})
+}
+
 type fakeUpstream struct {
 	mu     sync.Mutex
 	srv    *httptest.Server
@@ -143,6 +160,42 @@ var _ = Describe("DefaultProber", func() {
 		Expect(other.paths).To(BeEmpty())
 	})
 
+	It("warns once when a remote target has api_key_env set but no credential lookup is configured", func() {
+		var buf bytes.Buffer
+		captureXlog(&buf)
+
+		noLookup := NewProber(nil, nil)
+		c := proxied("argus-llm", "")
+		c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY"
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, up.srv.URL+"/v1/models", nil)
+		Expect(err).ToNot(HaveOccurred())
+
+		// authorize surfaces the misconfiguration as an error on every call
+		// (liveness/inference must not silently proceed unauthenticated)...
+		Expect(noLookup.authorize(req, c)).To(HaveOccurred())
+		Expect(noLookup.authorize(req, c)).To(HaveOccurred())
+
+		// ...but only logs the warning once per api_key_env, so a chain with
+		// no lookup configured does not spam the log on every probe tick.
+		Expect(strings.Count(buf.String(), "no credential lookup is configured")).To(Equal(1))
+		Expect(buf.String()).To(ContainSubstring("FAILOVER_PROBE_KEY"))
+	})
+
+	It("does not warn when a lookup is configured, even if the env var itself is unset", func() {
+		var buf bytes.Buffer
+		captureXlog(&buf)
+
+		c := proxied("argus-llm", "")
+		c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY_UNSET"
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, up.srv.URL+"/v1/models", nil)
+		Expect(err).ToNot(HaveOccurred())
+
+		// p (from BeforeEach) has a real lookup (os.Getenv); the env var is
+		// simply unset, which is a different, already-reported failure mode.
+		Expect(p.authorize(req, c)).To(HaveOccurred())
+		Expect(buf.String()).ToNot(ContainSubstring("no credential lookup is configured"))
+	})
+
 	DescribeTable("remote inference hits the usecase endpoint",
 		func(usecase, path string) {
 			Expect(p.Inference(ctx, proxied("m", "", usecase), KindRemote, false)).To(Succeed())

tests/e2e/e2e_suite_test.go

@@ -712,6 +712,11 @@ var _ = BeforeSuite(func() {
 		config.WithSystemState(systemState),
 		config.WithDebug(true),
 		config.WithGeneratedContentDir(generatedDir),
+		// Mirrors the CLI boundary (core/cli/run.go): the failover prober
+		// resolves api_key_env upstream credentials through this lookup.
+		// Without it, remote failover targets configured with api_key_env
+		// (e.g. the cloud-proxy chain-remote spec) never pass liveness.
+		config.WithProxyAPIKeyEnvLookup(os.Getenv),
 	)
 	Expect(err).ToNot(HaveOccurred())