diff --git a/core/http/endpoints/mcp/oauth2_transport.go b/core/http/endpoints/mcp/oauth2_transport.go index 8acc7eac4..9a6b3632a 100644 --- a/core/http/endpoints/mcp/oauth2_transport.go +++ b/core/http/endpoints/mcp/oauth2_transport.go @@ -131,17 +131,19 @@ func (rt *oauth2ClientCredentialsRoundTripper) currentToken(ctx context.Context) if rt.token != nil && now.Before(rt.refreshAt) { return rt.token, nil } - if rt.token != nil && now.Before(rt.nextAttempt) { + if now.Before(rt.nextAttempt) { if rt.stillUsable(now) { return rt.token, nil } - return nil, fmt.Errorf("oauth2: token for %s expired and refresh is failing", rt.conf.TokenURL) + return nil, fmt.Errorf("oauth2: token for %s is unavailable and refresh is failing", rt.conf.TokenURL) } fetchCtx, cancel := context.WithTimeout(context.WithValue(ctx, oauth2.HTTPClient, rt.tokenClient), oauth2TokenRequestTimeout) defer cancel() tok, err := rt.conf.Token(fetchCtx) if err != nil { + // A slow token request can consume the remaining safety margin. + now = rt.now() rt.nextAttempt = now.Add(oauth2RetryInterval) if rt.stillUsable(now) { xlog.Warn("MCP OAuth2 token refresh failed, keeping current token until near expiry", diff --git a/core/http/endpoints/mcp/oauth2_transport_test.go b/core/http/endpoints/mcp/oauth2_transport_test.go index 61c831a9d..4f8fa7811 100644 --- a/core/http/endpoints/mcp/oauth2_transport_test.go +++ b/core/http/endpoints/mcp/oauth2_transport_test.go @@ -19,6 +19,12 @@ import ( . "github.com/onsi/gomega" ) +type oauth2TestRoundTripper func(*http.Request) (*http.Response, error) + +func (f oauth2TestRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + // fakeIdP is a minimal OAuth2 token endpoint for the client_credentials grant. type fakeIdP struct { server *httptest.Server @@ -245,6 +251,57 @@ var _ = Describe("OAuth2 client_credentials for remote MCP servers", func() { Expect(rec.headers()[len(rec.headers())-1]).NotTo(Equal("Bearer tok-1")) }) + It("rechecks token expiry after a slow failed refresh", func() { + client := clientFor(baseCfg()) + Expect(doGet(client, rec.server.URL)).To(Succeed()) + rt := client.Transport.(*oauth2ClientCredentialsRoundTripper) + base := rt.tokenClient.Transport + rt.tokenClient.Transport = oauth2TestRoundTripper(func(r *http.Request) (*http.Response, error) { + clock.Advance(20 * time.Second) + return base.RoundTrip(r) + }) + idp.failing.Store(true) + clock.Advance(51 * time.Second) + + Expect(doGet(client, rec.server.URL)).NotTo(Succeed()) + Expect(rec.headers()).To(Equal([]string{"Bearer tok-1"}), "the failed fetch crossed the expiry safety margin") + }) + + It("starts the retry interval when a slow failed refresh finishes", func() { + client := clientFor(baseCfg()) + Expect(doGet(client, rec.server.URL)).To(Succeed()) + rt := client.Transport.(*oauth2ClientCredentialsRoundTripper) + base := rt.tokenClient.Transport + rt.tokenClient.Transport = oauth2TestRoundTripper(func(r *http.Request) (*http.Response, error) { + clock.Advance(10 * time.Second) + return base.RoundTrip(r) + }) + idp.failing.Store(true) + clock.Advance(51 * time.Second) + + Expect(doGet(client, rec.server.URL)).To(Succeed()) + attempts := idp.requests.Load() + Expect(doGet(client, rec.server.URL)).To(Succeed()) + Expect(idp.requests.Load()).To(Equal(attempts)) + }) + + It("throttles failed fetches before any token is cached", func() { + client := clientFor(baseCfg()) + idp.failing.Store(true) + Expect(doGet(client, rec.server.URL)).NotTo(Succeed()) + attempts := idp.requests.Load() + Expect(attempts).To(BeNumerically(">", 0)) + + Expect(doGet(client, rec.server.URL)).NotTo(Succeed()) + Expect(idp.requests.Load()).To(Equal(attempts)) + Expect(rec.headers()).To(BeEmpty()) + + idp.failing.Store(false) + clock.Advance(oauth2RetryInterval) + Expect(doGet(client, rec.server.URL)).To(Succeed()) + Expect(rec.headers()).To(HaveLen(1)) + }) + It("errors instead of sending an unauthenticated request when the first fetch fails", func() { cfg := baseCfg() cfg.ClientSecret = "wrong" diff --git a/docs/content/features/mcp.md b/docs/content/features/mcp.md index da697fab9..94fa35f66 100644 --- a/docs/content/features/mcp.md +++ b/docs/content/features/mcp.md @@ -137,8 +137,9 @@ Behaviour: - The token is fetched on the first request to the server, cached, and sent as `Authorization: Bearer ` on every request, for both streamable HTTP and SSE transports. - It is refreshed proactively once **half** of its lifetime (`expires_in`) has elapsed. If the provider omits `expires_in`, a lifetime of 5 minutes is assumed. -- If a refresh fails, the current token keeps being used until 30 seconds before it expires (refresh is retried at most every 5 seconds). After that, requests to the MCP server fail with an error; LocalAI never falls back to an unauthenticated request. +- If a refresh fails, the current token keeps being used until 30 seconds before it expires (retries wait at least 5 seconds after a failed fetch, including the first fetch). After that, requests to the MCP server fail with an error; LocalAI never falls back to an unauthenticated request. - Configuring both `token` and `oauth2` for the same server is rejected when the model configuration is loaded, as are missing or duplicated `client_id`/`client_secret` sources. An environment variable named in `*_env` that is unset or empty makes the connection to that server fail, and the server is reported with an error in the server listing. +- Token caches and retry timers are local to each connection-owning process. Workers obtain and refresh their own tokens; this cache is not shared across the cluster. - The environment variables are read by the process that connects to the MCP server. In distributed mode that is the agent worker, so set them there. Servers configured with a static `token` behave exactly as before.