From c5997fd71a3728011f5521bb48f97fdb979df71b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:03:58 +0000 Subject: [PATCH] fix(mcp): check token expiry after failed refresh A slow failed refresh can cross the token's safety margin. Check the current time before reusing the token and start retry backoff when the failed fetch finishes. Apply backoff before the first successful fetch too. Add regression tests for these cases and document per-process caches. Assisted-by: Codex:GPT-6 (cherry picked from commit a849e872ec88730ae3048bb21232db379293568a) Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/mcp/oauth2_transport.go | 6 +- .../endpoints/mcp/oauth2_transport_test.go | 57 +++++++++++++++++++ docs/content/features/mcp.md | 3 +- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/core/http/endpoints/mcp/oauth2_transport.go b/core/http/endpoints/mcp/oauth2_transport.go index 8acc7eac4..9a6b3632a 100644 --- a/core/http/endpoints/mcp/oauth2_transport.go +++ b/core/http/endpoints/mcp/oauth2_transport.go @@ -131,17 +131,19 @@ func (rt *oauth2ClientCredentialsRoundTripper) currentToken(ctx context.Context) if rt.token != nil && now.Before(rt.refreshAt) { return rt.token, nil } - if rt.token != nil && now.Before(rt.nextAttempt) { + if now.Before(rt.nextAttempt) { if rt.stillUsable(now) { return rt.token, nil } - return nil, fmt.Errorf("oauth2: token for %s expired and refresh is failing", rt.conf.TokenURL) + return nil, fmt.Errorf("oauth2: token for %s is unavailable and refresh is failing", rt.conf.TokenURL) } fetchCtx, cancel := context.WithTimeout(context.WithValue(ctx, oauth2.HTTPClient, rt.tokenClient), oauth2TokenRequestTimeout) defer cancel() tok, err := rt.conf.Token(fetchCtx) if err != nil { + // A slow token request can consume the remaining safety margin. + now = rt.now() rt.nextAttempt = now.Add(oauth2RetryInterval) if rt.stillUsable(now) { xlog.Warn("MCP OAuth2 token refresh failed, keeping current token until near expiry", diff --git a/core/http/endpoints/mcp/oauth2_transport_test.go b/core/http/endpoints/mcp/oauth2_transport_test.go index 61c831a9d..4f8fa7811 100644 --- a/core/http/endpoints/mcp/oauth2_transport_test.go +++ b/core/http/endpoints/mcp/oauth2_transport_test.go @@ -19,6 +19,12 @@ import ( . "github.com/onsi/gomega" ) +type oauth2TestRoundTripper func(*http.Request) (*http.Response, error) + +func (f oauth2TestRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + // fakeIdP is a minimal OAuth2 token endpoint for the client_credentials grant. type fakeIdP struct { server *httptest.Server @@ -245,6 +251,57 @@ var _ = Describe("OAuth2 client_credentials for remote MCP servers", func() { Expect(rec.headers()[len(rec.headers())-1]).NotTo(Equal("Bearer tok-1")) }) + It("rechecks token expiry after a slow failed refresh", func() { + client := clientFor(baseCfg()) + Expect(doGet(client, rec.server.URL)).To(Succeed()) + rt := client.Transport.(*oauth2ClientCredentialsRoundTripper) + base := rt.tokenClient.Transport + rt.tokenClient.Transport = oauth2TestRoundTripper(func(r *http.Request) (*http.Response, error) { + clock.Advance(20 * time.Second) + return base.RoundTrip(r) + }) + idp.failing.Store(true) + clock.Advance(51 * time.Second) + + Expect(doGet(client, rec.server.URL)).NotTo(Succeed()) + Expect(rec.headers()).To(Equal([]string{"Bearer tok-1"}), "the failed fetch crossed the expiry safety margin") + }) + + It("starts the retry interval when a slow failed refresh finishes", func() { + client := clientFor(baseCfg()) + Expect(doGet(client, rec.server.URL)).To(Succeed()) + rt := client.Transport.(*oauth2ClientCredentialsRoundTripper) + base := rt.tokenClient.Transport + rt.tokenClient.Transport = oauth2TestRoundTripper(func(r *http.Request) (*http.Response, error) { + clock.Advance(10 * time.Second) + return base.RoundTrip(r) + }) + idp.failing.Store(true) + clock.Advance(51 * time.Second) + + Expect(doGet(client, rec.server.URL)).To(Succeed()) + attempts := idp.requests.Load() + Expect(doGet(client, rec.server.URL)).To(Succeed()) + Expect(idp.requests.Load()).To(Equal(attempts)) + }) + + It("throttles failed fetches before any token is cached", func() { + client := clientFor(baseCfg()) + idp.failing.Store(true) + Expect(doGet(client, rec.server.URL)).NotTo(Succeed()) + attempts := idp.requests.Load() + Expect(attempts).To(BeNumerically(">", 0)) + + Expect(doGet(client, rec.server.URL)).NotTo(Succeed()) + Expect(idp.requests.Load()).To(Equal(attempts)) + Expect(rec.headers()).To(BeEmpty()) + + idp.failing.Store(false) + clock.Advance(oauth2RetryInterval) + Expect(doGet(client, rec.server.URL)).To(Succeed()) + Expect(rec.headers()).To(HaveLen(1)) + }) + It("errors instead of sending an unauthenticated request when the first fetch fails", func() { cfg := baseCfg() cfg.ClientSecret = "wrong" diff --git a/docs/content/features/mcp.md b/docs/content/features/mcp.md index da697fab9..94fa35f66 100644 --- a/docs/content/features/mcp.md +++ b/docs/content/features/mcp.md @@ -137,8 +137,9 @@ Behaviour: - The token is fetched on the first request to the server, cached, and sent as `Authorization: Bearer ` on every request, for both streamable HTTP and SSE transports. - It is refreshed proactively once **half** of its lifetime (`expires_in`) has elapsed. If the provider omits `expires_in`, a lifetime of 5 minutes is assumed. -- If a refresh fails, the current token keeps being used until 30 seconds before it expires (refresh is retried at most every 5 seconds). After that, requests to the MCP server fail with an error; LocalAI never falls back to an unauthenticated request. +- If a refresh fails, the current token keeps being used until 30 seconds before it expires (retries wait at least 5 seconds after a failed fetch, including the first fetch). After that, requests to the MCP server fail with an error; LocalAI never falls back to an unauthenticated request. - Configuring both `token` and `oauth2` for the same server is rejected when the model configuration is loaded, as are missing or duplicated `client_id`/`client_secret` sources. An environment variable named in `*_env` that is unset or empty makes the connection to that server fail, and the server is reported with an error in the server listing. +- Token caches and retry timers are local to each connection-owning process. Workers obtain and refresh their own tokens; this cache is not shared across the cluster. - The environment variables are read by the process that connects to the MCP server. In distributed mode that is the agent worker, so set them there. Servers configured with a static `token` behave exactly as before.