From 55f7d5bfdfc159d0a5a34254b3fa1591be1fdbe8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 27 Sep 2026 02:54:13 +0000 Subject: [PATCH] fix(e2e): wire proxy api_key_env lookup into the e2e in-process app 19f6a18b8 moved credential-env resolution behind an explicit lookup (ApplicationConfig.ProxyAPIKeyEnvLookup / config.WithProxyAPIKeyEnvLookup), wired only at the CLI boundary (core/cli/run.go). The e2e suite builds its Application in-process without that option, so the failover prober could never resolve a remote target's api_key_env, remote liveness never passed, and "fails over ... and fails back" hung waiting for chain-remote to recover. Pass config.WithProxyAPIKeyEnvLookup(os.Getenv) there too, same as the CLI. worker/federated commands don't serve proxy/failover configs and tests/e2e-ui never sets api_key_env, so neither needs the lookup. Also make the misconfiguration itself easier to diagnose: the prober now logs a one-time xlog.Warn per api_key_env when a remote target sets it but no lookup is configured, instead of only surfacing it as a per-probe "is unset" error indistinguishable from a genuinely empty env var. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Ettore Di Giacinto --- core/services/failover/prober.go | 12 ++++++ core/services/failover/prober_test.go | 53 +++++++++++++++++++++++++++ tests/e2e/e2e_suite_test.go | 5 +++ 3 files changed, 70 insertions(+) diff --git a/core/services/failover/prober.go b/core/services/failover/prober.go index 6b27e4f8a..cb7a53183 100644 --- a/core/services/failover/prober.go +++ b/core/services/failover/prober.go @@ -12,10 +12,12 @@ import ( "net/http" "net/url" "strings" + "sync" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" ) // ErrNotLoaded is what Inference returns for a local target whose backend is @@ -35,6 +37,11 @@ type DefaultProber struct { HTTP *http.Client Loaded LoadedFunc EnvLookup func(string) string + + // unresolvableEnvWarned tracks which api_key_env names already got the + // "no lookup configured" warning below, so a chain with no working + // credential lookup does not re-log on every probe tick. + unresolvableEnvWarned sync.Map // map[string]struct{} } func NewProber(loaded LoadedFunc, envLookup func(string) string) *DefaultProber { @@ -103,6 +110,11 @@ func PrepareTarget(cfg *config.ModelConfig) { } func (p *DefaultProber) authorize(req *http.Request, cfg config.ModelConfig) error { + if cfg.Proxy.APIKeyEnv != "" && p.EnvLookup == nil { + if _, warned := p.unresolvableEnvWarned.LoadOrStore(cfg.Proxy.APIKeyEnv, struct{}{}); !warned { + xlog.Warn("failover: remote target has api_key_env set but no credential lookup is configured; liveness and requests will fail authorization", "target", cfg.Name, "api_key_env", cfg.Proxy.APIKeyEnv) + } + } key, err := cfg.Proxy.ResolveAPIKey(p.EnvLookup) if err != nil || key == "" { return err diff --git a/core/services/failover/prober_test.go b/core/services/failover/prober_test.go index fd22bd1e3..8e66d2be3 100644 --- a/core/services/failover/prober_test.go +++ b/core/services/failover/prober_test.go @@ -1,23 +1,40 @@ package failover import ( + "bytes" "context" "encoding/json" "errors" "io" + "log/slog" "net/http" "net/http/httptest" "os" + "strings" "sync" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ggrpc "google.golang.org/grpc" ) +// captureXlog redirects the package-wide xlog logger to buf for the duration +// of a test and restores the suite's default on cleanup. xlog exposes no +// getter for the current logger, so this restores the same default the +// entrypoint installs rather than the prior value (same pattern as +// core/config/model_artifact_fallback_test.go). +func captureXlog(buf *bytes.Buffer) { + handler := slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelWarn}) + xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelWarn)) + DeferCleanup(func() { + xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text")) + }) +} + type fakeUpstream struct { mu sync.Mutex srv *httptest.Server @@ -143,6 +160,42 @@ var _ = Describe("DefaultProber", func() { Expect(other.paths).To(BeEmpty()) }) + It("warns once when a remote target has api_key_env set but no credential lookup is configured", func() { + var buf bytes.Buffer + captureXlog(&buf) + + noLookup := NewProber(nil, nil) + c := proxied("argus-llm", "") + c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, up.srv.URL+"/v1/models", nil) + Expect(err).ToNot(HaveOccurred()) + + // authorize surfaces the misconfiguration as an error on every call + // (liveness/inference must not silently proceed unauthenticated)... + Expect(noLookup.authorize(req, c)).To(HaveOccurred()) + Expect(noLookup.authorize(req, c)).To(HaveOccurred()) + + // ...but only logs the warning once per api_key_env, so a chain with + // no lookup configured does not spam the log on every probe tick. + Expect(strings.Count(buf.String(), "no credential lookup is configured")).To(Equal(1)) + Expect(buf.String()).To(ContainSubstring("FAILOVER_PROBE_KEY")) + }) + + It("does not warn when a lookup is configured, even if the env var itself is unset", func() { + var buf bytes.Buffer + captureXlog(&buf) + + c := proxied("argus-llm", "") + c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY_UNSET" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, up.srv.URL+"/v1/models", nil) + Expect(err).ToNot(HaveOccurred()) + + // p (from BeforeEach) has a real lookup (os.Getenv); the env var is + // simply unset, which is a different, already-reported failure mode. + Expect(p.authorize(req, c)).To(HaveOccurred()) + Expect(buf.String()).ToNot(ContainSubstring("no credential lookup is configured")) + }) + DescribeTable("remote inference hits the usecase endpoint", func(usecase, path string) { Expect(p.Inference(ctx, proxied("m", "", usecase), KindRemote, false)).To(Succeed()) diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index 0d1f51608..6cdf84f2b 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -712,6 +712,11 @@ var _ = BeforeSuite(func() { config.WithSystemState(systemState), config.WithDebug(true), config.WithGeneratedContentDir(generatedDir), + // Mirrors the CLI boundary (core/cli/run.go): the failover prober + // resolves api_key_env upstream credentials through this lookup. + // Without it, remote failover targets configured with api_key_env + // (e.g. the cloud-proxy chain-remote spec) never pass liveness. + config.WithProxyAPIKeyEnvLookup(os.Getenv), ) Expect(err).ToNot(HaveOccurred())