Fix double-caching of OAuth tokens in Azure client secret credentials (#1573)

## Summary

Fixes the double-caching of OAuth tokens in Azure client secret
credentials (#1549) and unifies the internal token source infrastructure
on `auth.TokenSource`.

## Why

`AzureClientSecretCredentials.tokenSourceFor` called
`clientcredentials.Config{}.TokenSource(ctx)`, which returns an
`oauth2.ReuseTokenSource` with a hardcoded 10-second `expiryDelta`. This
token source was then wrapped in `azureReuseTokenSource` (a
`cachedTokenSource`) and again in `serviceToServiceVisitor` (another
`cachedTokenSource`). The inner `ReuseTokenSource` swallowed proactive
refresh attempts from the outer layers -- it kept returning its cached
token until only ~10 seconds remained before expiry, defeating the
20-minute async refresh window and causing bursts of 401s at token
expiry.

This is the same class of bug as #1550 (M2M OAuth), but in the Azure
client secret path.

## What changed

### Interface changes

None. All changes are to unexported types and functions.

### Behavioral changes

- `AzureClientSecretCredentials` no longer double-caches tokens. Each
call to the underlying token source results in an HTTP request to the
token endpoint; caching is purely controlled by the outer
`azureReuseTokenSource` and `serviceToServiceVisitor` layers.

### Internal changes

- **`azureHostResolver` interface** now returns `auth.TokenSource`
instead of `oauth2.TokenSource`, and no longer takes a `context.Context`
parameter. Context is passed per-call via `Token(ctx)`.
- **`serviceToServiceVisitor` and `refreshableVisitor`** now accept
`auth.TokenSource` directly, removing the `authconv.AuthTokenSource`
wrapping at call sites.
- **`azureReuseTokenSource` and `wrap`** operate on `auth.TokenSource`
instead of `oauth2.TokenSource`, eliminating the `authconv` round-trip.
- **`AzureClientSecretCredentials`**: removed the duplicate
`tokenSourceFor` (which used `clientcredentials.Config.TokenSource` with
inner caching). The former `authTokenSourceFor` -- which calls
`clientcredentials.Config.Token(ctx)` without inner caching -- is now
the sole `tokenSourceFor`.
- **`AzureCliCredentials.tokenSourceFor`**: returns an
`auth.TokenSource` that creates a fresh `azureCliTokenSource` per call,
properly threading the caller's context.
- **`AzureMsiCredentials.tokenSourceFor`**: wraps
`NewAzureMsiTokenSource` with `authconv.AuthTokenSource`.
- **GCP and M2M credential providers**: callers wrap their
`oauth2.TokenSource` with `authconv.AuthTokenSource` before passing to
the visitor functions.

## How is this tested?

- `TestAzureClientSecretCredentials_tokenSourceFor_noCaching`: calls
`Token()` N times and asserts N HTTP requests hit the token endpoint.
Fails when the fix is reverted (inner caching collapses N calls into 1).
- `TestAzureClientSecretCredentials_Configure`: happy-path test for the
full Azure client secret credential flow.
- All existing `config/` tests pass.

Signed-off-by: Ubuntu <renaud.hartert@databricks.com>

Author: renaudhartert-db

NEXT_CHANGELOG.md

@@ -13,6 +13,7 @@
 
 ### Bug Fixes
 
+ * Fix double-caching of OAuth tokens in Azure client secret credentials ([#1549](https://github.com/databricks/databricks-sdk-go/issues/1549)).
  * Disable async token refresh for GCP credential providers to avoid wasted refresh attempts caused by double-caching with Google's internal `oauth2.ReuseTokenSource` ([#1549](https://github.com/databricks/databricks-sdk-go/issues/1549)).
 
 ### Documentation

config/auth_azure_client_secret.go

@@ -5,11 +5,11 @@ import (
 	"fmt"
 	"net/url"
 
+	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 
 	"github.com/databricks/databricks-sdk-go/config/credentials"
 	"github.com/databricks/databricks-sdk-go/config/experimental/auth"
-	"github.com/databricks/databricks-sdk-go/config/experimental/auth/authconv"
 	"github.com/databricks/databricks-sdk-go/logger"
 )
 
@@ -20,16 +20,24 @@ func (c AzureClientSecretCredentials) Name() string {
 	return "azure-client-secret"
 }
 
+// tokenSourceFor returns an auth.TokenSource that creates a fresh
+// clientcredentials.Config on each call and uses the provided context for the
+// token exchange. This does not include the inner caching layer from
+// clientcredentials.Config.TokenSource so that caching is purely controlled
+// upstream.
 func (c AzureClientSecretCredentials) tokenSourceFor(
-	ctx context.Context, cfg *Config, aadEndpoint, resource string) auth.TokenSource {
-	return authconv.AuthTokenSource((&clientcredentials.Config{
-		ClientID:     cfg.AzureClientID,
-		ClientSecret: cfg.AzureClientSecret,
-		TokenURL:     fmt.Sprintf("%s%s/oauth2/token", aadEndpoint, cfg.AzureTenantID),
-		EndpointParams: url.Values{
-			"resource": []string{resource},
-		},
-	}).TokenSource(ctx))
+	_ context.Context, cfg *Config, aadEndpoint, resource string) auth.TokenSource {
+	return auth.TokenSourceFn(func(ctx context.Context) (*oauth2.Token, error) {
+		ctx = cfg.refreshClient.InContextForOAuth2(ctx)
+		return (&clientcredentials.Config{
+			ClientID:     cfg.AzureClientID,
+			ClientSecret: cfg.AzureClientSecret,
+			TokenURL:     fmt.Sprintf("%s%s/oauth2/token", aadEndpoint, cfg.AzureTenantID),
+			EndpointParams: url.Values{
+				"resource": []string{resource},
+			},
+		}).Token(ctx)
+	})
 }
 
 // TODO: We need to expose which authentication mechanism is used to Terraform,

config/auth_azure_client_secret_test.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/databricks/databricks-sdk-go/httpclient"
+	"github.com/databricks/databricks-sdk-go/httpclient/fixtures"
+)
+
+// tokenCountingTransport wraps an http.RoundTripper and counts the number of
+// round trips made. It is used to verify that the token source does not cache
+// tokens internally, ensuring that each Token() call results in an HTTP
+// request.
+type tokenCountingTransport struct {
+	inner http.RoundTripper
+	count atomic.Int32
+}
+
+func (t *tokenCountingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	t.count.Add(1)
+	return t.inner.RoundTrip(req)
+}
+
+func (t *tokenCountingTransport) SkipRetryOnIO() bool {
+	return true
+}
+
+// TestAzureClientSecretCredentials_tokenSourceFor_noCaching verifies that
+// the auth.TokenSource returned by tokenSourceFor does not cache tokens
+// internally. Before the fix, the code used clientcredentials.Config.TokenSource
+// which returns an oauth2.ReuseTokenSource with a 10s expiryDelta. This inner
+// cache swallowed proactive refresh attempts from the outer caching layers
+// (azureReuseTokenSource and serviceToServiceVisitor), causing bursts
+// of 401s at token expiry.
+// See https://github.com/databricks/databricks-sdk-go/issues/1549.
+func TestAzureClientSecretCredentials_tokenSourceFor_noCaching(t *testing.T) {
+	counter := &tokenCountingTransport{
+		inner: fixtures.MappingTransport{
+			"POST /test-tenant-id/oauth2/token": {
+				Response: map[string]any{
+					"token_type":   "Bearer",
+					"access_token": "test-token",
+					"expires_in":   3600,
+				},
+			},
+		},
+	}
+
+	cfg := &Config{
+		AzureClientID:     "test-client-id",
+		AzureClientSecret: "test-client-secret",
+		AzureTenantID:     "test-tenant-id",
+	}
+	cfg.refreshClient = httpclient.NewApiClient(httpclient.ClientConfig{
+		Transport: counter,
+	})
+
+	c := AzureClientSecretCredentials{}
+	ts := c.tokenSourceFor(context.Background(), cfg, "https://login.microsoftonline.com/", "https://management.azure.com/")
+
+	const numCalls = 3
+	for i := range numCalls {
+		tok, err := ts.Token(context.Background())
+		if err != nil {
+			t.Fatalf("Token() call %d: unexpected error: %v", i+1, err)
+		}
+		if tok.AccessToken != "test-token" {
+			t.Errorf("Token() call %d: got access token %q, want %q", i+1, tok.AccessToken, "test-token")
+		}
+	}
+
+	if got := int(counter.count.Load()); got != numCalls {
+		t.Errorf("token endpoint was called %d times, want %d (inner caching is present)", got, numCalls)
+	}
+}
+
+// TestAzureClientSecretCredentials_Configure verifies the happy path of the
+// full Azure client secret credential flow, including both the inner
+// (workspace) token and the management token.
+func TestAzureClientSecretCredentials_Configure(t *testing.T) {
+	tokenExpiry := time.Now().Add(time.Hour).Unix()
+	assertHeaders(t, &Config{
+		Host:              "https://adb-123.azuredatabricks.net",
+		AzureClientID:     "test-client-id",
+		AzureClientSecret: "test-client-secret",
+		AzureTenantID:     "test-tenant-id",
+		AuthType:          "azure-client-secret",
+		HTTPTransport: fixtures.MappingTransport{
+			"POST /test-tenant-id/oauth2/token": {
+				Response: map[string]any{
+					"token_type":   "Bearer",
+					"access_token": "workspace-token",
+					"expires_on":   fmt.Sprintf("%d", tokenExpiry),
+				},
+			},
+		},
+	}, map[string]string{
+		"Authorization":                          "Bearer workspace-token",
+		"X-Databricks-Azure-Sp-Management-Token": "workspace-token",
+	})
+}