fix(mcp): check token expiry after failed refresh

A slow failed refresh can cross the token's safety margin.
Check the current time before reusing the token and start retry
backoff when the failed fetch finishes. Apply backoff before the
first successful fetch too.

Add regression tests for these cases and document per-process caches.

Assisted-by: Codex:GPT-6
(cherry picked from commit a849e872ec)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/http/endpoints/mcp/oauth2_transport.go

@@ -131,17 +131,19 @@ func (rt *oauth2ClientCredentialsRoundTripper) currentToken(ctx context.Context)
 	if rt.token != nil && now.Before(rt.refreshAt) {
 		return rt.token, nil
 	}
-	if rt.token != nil && now.Before(rt.nextAttempt) {
+	if now.Before(rt.nextAttempt) {
 		if rt.stillUsable(now) {
 			return rt.token, nil
 		}
-		return nil, fmt.Errorf("oauth2: token for %s expired and refresh is failing", rt.conf.TokenURL)
+		return nil, fmt.Errorf("oauth2: token for %s is unavailable and refresh is failing", rt.conf.TokenURL)
 	}
 
 	fetchCtx, cancel := context.WithTimeout(context.WithValue(ctx, oauth2.HTTPClient, rt.tokenClient), oauth2TokenRequestTimeout)
 	defer cancel()
 	tok, err := rt.conf.Token(fetchCtx)
 	if err != nil {
+		// A slow token request can consume the remaining safety margin.
+		now = rt.now()
 		rt.nextAttempt = now.Add(oauth2RetryInterval)
 		if rt.stillUsable(now) {
 			xlog.Warn("MCP OAuth2 token refresh failed, keeping current token until near expiry",

core/http/endpoints/mcp/oauth2_transport_test.go

@@ -19,6 +19,12 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+type oauth2TestRoundTripper func(*http.Request) (*http.Response, error)
+
+func (f oauth2TestRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
 // fakeIdP is a minimal OAuth2 token endpoint for the client_credentials grant.
 type fakeIdP struct {
 	server    *httptest.Server
@@ -245,6 +251,57 @@ var _ = Describe("OAuth2 client_credentials for remote MCP servers", func() {
 		Expect(rec.headers()[len(rec.headers())-1]).NotTo(Equal("Bearer tok-1"))
 	})
 
+	It("rechecks token expiry after a slow failed refresh", func() {
+		client := clientFor(baseCfg())
+		Expect(doGet(client, rec.server.URL)).To(Succeed())
+		rt := client.Transport.(*oauth2ClientCredentialsRoundTripper)
+		base := rt.tokenClient.Transport
+		rt.tokenClient.Transport = oauth2TestRoundTripper(func(r *http.Request) (*http.Response, error) {
+			clock.Advance(20 * time.Second)
+			return base.RoundTrip(r)
+		})
+		idp.failing.Store(true)
+		clock.Advance(51 * time.Second)
+
+		Expect(doGet(client, rec.server.URL)).NotTo(Succeed())
+		Expect(rec.headers()).To(Equal([]string{"Bearer tok-1"}), "the failed fetch crossed the expiry safety margin")
+	})
+
+	It("starts the retry interval when a slow failed refresh finishes", func() {
+		client := clientFor(baseCfg())
+		Expect(doGet(client, rec.server.URL)).To(Succeed())
+		rt := client.Transport.(*oauth2ClientCredentialsRoundTripper)
+		base := rt.tokenClient.Transport
+		rt.tokenClient.Transport = oauth2TestRoundTripper(func(r *http.Request) (*http.Response, error) {
+			clock.Advance(10 * time.Second)
+			return base.RoundTrip(r)
+		})
+		idp.failing.Store(true)
+		clock.Advance(51 * time.Second)
+
+		Expect(doGet(client, rec.server.URL)).To(Succeed())
+		attempts := idp.requests.Load()
+		Expect(doGet(client, rec.server.URL)).To(Succeed())
+		Expect(idp.requests.Load()).To(Equal(attempts))
+	})
+
+	It("throttles failed fetches before any token is cached", func() {
+		client := clientFor(baseCfg())
+		idp.failing.Store(true)
+		Expect(doGet(client, rec.server.URL)).NotTo(Succeed())
+		attempts := idp.requests.Load()
+		Expect(attempts).To(BeNumerically(">", 0))
+
+		Expect(doGet(client, rec.server.URL)).NotTo(Succeed())
+		Expect(idp.requests.Load()).To(Equal(attempts))
+		Expect(rec.headers()).To(BeEmpty())
+
+		idp.failing.Store(false)
+		clock.Advance(oauth2RetryInterval)
+		Expect(doGet(client, rec.server.URL)).To(Succeed())
+		Expect(rec.headers()).To(HaveLen(1))
+	})
+
 	It("errors instead of sending an unauthenticated request when the first fetch fails", func() {
 		cfg := baseCfg()
 		cfg.ClientSecret = "wrong"

docs/content/features/mcp.md

@@ -137,8 +137,9 @@ Behaviour:
 
 - The token is fetched on the first request to the server, cached, and sent as `Authorization: Bearer <token>` on every request, for both streamable HTTP and SSE transports.
 - It is refreshed proactively once **half** of its lifetime (`expires_in`) has elapsed. If the provider omits `expires_in`, a lifetime of 5 minutes is assumed.
-- If a refresh fails, the current token keeps being used until 30 seconds before it expires (refresh is retried at most every 5 seconds). After that, requests to the MCP server fail with an error; LocalAI never falls back to an unauthenticated request.
+- If a refresh fails, the current token keeps being used until 30 seconds before it expires (retries wait at least 5 seconds after a failed fetch, including the first fetch). After that, requests to the MCP server fail with an error; LocalAI never falls back to an unauthenticated request.
 - Configuring both `token` and `oauth2` for the same server is rejected when the model configuration is loaded, as are missing or duplicated `client_id`/`client_secret` sources. An environment variable named in `*_env` that is unset or empty makes the connection to that server fail, and the server is reported with an error in the server listing.
+- Token caches and retry timers are local to each connection-owning process. Workers obtain and refresh their own tokens; this cache is not shared across the cluster.
 - The environment variables are read by the process that connects to the MCP server. In distributed mode that is the agent worker, so set them there.
 
 Servers configured with a static `token` behave exactly as before.