Add --force-refresh flag to CLI token source

Pass --force-refresh to the Databricks CLI auth token command to bypass
the CLI's internal token cache. The SDK manages its own token caching,
so the CLI serving stale tokens from its cache causes unnecessary
refresh failures.

Commands are now tried in order: forceCmd (--profile + --force-refresh),
profileCmd (--profile), hostCmd (--host), falling back progressively
for older CLI versions that don't support newer flags.

Signed-off-by: Mihai Mitrea <mihai.mitrea@databricks.com>

Author: mihaimitrea-db

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 
 ### New Features and Improvements
 
+* Pass `--force-refresh` to Databricks CLI `auth token` command to bypass the CLI's internal token cache.
 * Added `HostMetadataResolver` hook to allow callers to customize host metadata resolution, e.g. with caching ([#1572](https://github.com/databricks/databricks-sdk-go/pull/1572)).
 * Added `NewLimitIterator` to `listing` package for lazy iteration with a cap on output items ([#1555](https://github.com/databricks/databricks-sdk-go/pull/1555)).
 

config/cli_token_source.go

@@ -31,12 +31,19 @@ type cliTokenResponse struct {
 	Expiry      string `json:"expiry"`
 }
 
+// CliTokenSource fetches OAuth tokens by shelling out to the Databricks CLI.
+// Commands are tried in order: forceCmd -> profileCmd -> hostCmd, progressively
+// falling back to simpler invocations for older CLI versions.
 type CliTokenSource struct {
-	// cmd is the primary command to execute (--profile when available, --host otherwise).
-	cmd []string
+	// forceCmd uses --profile with --force-refresh to bypass the CLI's token cache.
+	// Nil when cfg.Profile is empty (--force-refresh requires --profile support).
+	forceCmd []string
 
-	// hostCmd is a fallback command using --host, used when the primary --profile
-	// command fails because the CLI is too old to support --profile.
+	// profileCmd uses --profile for token lookup. Nil when cfg.Profile is empty.
+	profileCmd []string
+
+	// hostCmd uses --host as a fallback for CLIs that predate --profile support.
+	// Nil when cfg.Host is empty.
 	hostCmd []string
 }
 
@@ -45,25 +52,25 @@ func NewCliTokenSource(cfg *Config) (*CliTokenSource, error) {
 	if err != nil {
 		return nil, err
 	}
-	profileCmd, hostCmd := buildCliCommands(cliPath, cfg)
-	return &CliTokenSource{cmd: profileCmd, hostCmd: hostCmd}, nil
+	forceCmd, profileCmd, hostCmd := buildCliCommands(cliPath, cfg)
+	return &CliTokenSource{forceCmd: forceCmd, profileCmd: profileCmd, hostCmd: hostCmd}, nil
 }
 
 // buildCliCommands constructs the CLI commands for fetching an auth token.
-// When cfg.Profile is set, the primary command uses --profile and a fallback
-// --host command is also returned for compatibility with older CLIs.
-// When cfg.Profile is empty, the primary command uses --host and no fallback
-// is needed.
-func buildCliCommands(cliPath string, cfg *Config) (primaryCmd []string, hostCmd []string) {
+// When cfg.Profile is set, three commands are built: a --force-refresh variant,
+// a plain --profile variant, and (when host is available) a --host fallback.
+// When cfg.Profile is empty, only --host is returned — the CLI must support
+// --profile before --force-refresh can be used (monotonic feature assumption).
+func buildCliCommands(cliPath string, cfg *Config) ([]string, []string, []string) {
+	var forceCmd, profileCmd, hostCmd []string
 	if cfg.Profile != "" {
-		primary := []string{cliPath, "auth", "token", "--profile", cfg.Profile}
-		if cfg.Host != "" {
-			// Build a --host fallback for old CLIs that don't support --profile.
-			return primary, buildHostCommand(cliPath, cfg)
-		}
-		return primary, nil
+		profileCmd = []string{cliPath, "auth", "token", "--profile", cfg.Profile}
+		forceCmd = append(profileCmd, "--force-refresh")
 	}
-	return buildHostCommand(cliPath, cfg), nil
+	if cfg.Host != "" {
+		hostCmd = buildHostCommand(cliPath, cfg)
+	}
+	return forceCmd, profileCmd, hostCmd
 }
 
 // buildHostCommand constructs the legacy --host based CLI command.
@@ -77,16 +84,36 @@ func buildHostCommand(cliPath string, cfg *Config) []string {
 }
 
 // Token fetches an OAuth token by shelling out to the Databricks CLI.
-// When a --profile command is configured, it is tried first. If the CLI
-// returns "unknown flag: --profile" (indicating an older CLI version),
-// the fallback --host command is used instead.
+// Commands are tried in order: forceCmd -> profileCmd -> hostCmd, skipping nil
+// entries. Each command falls through to the next on "unknown flag" errors,
+// logging a warning about the unsupported feature.
 func (c *CliTokenSource) Token(ctx context.Context) (*oauth2.Token, error) {
-	tok, err := c.execCliCommand(ctx, c.cmd)
-	if err != nil && c.hostCmd != nil && isUnknownFlagError(err) {
+	if c.forceCmd != nil {
+		tok, err := c.execCliCommand(ctx, c.forceCmd)
+		if err == nil {
+			return tok, nil
+		}
+		if !isUnknownFlagError(err, "--force-refresh") && !isUnknownFlagError(err, "--profile") {
+			return nil, err
+		}
+		logger.Warnf(ctx, "Databricks CLI does not support --force-refresh flag. The CLI's token cache may provide stale tokens. Please upgrade your CLI to the latest version.")
+	}
+
+	if c.profileCmd != nil {
+		tok, err := c.execCliCommand(ctx, c.profileCmd)
+		if err == nil {
+			return tok, nil
+		}
+		if !isUnknownFlagError(err, "--profile") {
+			return nil, err
+		}
 		logger.Warnf(ctx, "Databricks CLI does not support --profile flag. Falling back to --host. Please upgrade your CLI to the latest version.")
-		return c.execCliCommand(ctx, c.hostCmd)
 	}
-	return tok, err
+
+	if c.hostCmd == nil {
+		return nil, fmt.Errorf("cannot get access token: no CLI commands available")
+	}
+	return c.execCliCommand(ctx, c.hostCmd)
 }
 
 func (c *CliTokenSource) execCliCommand(ctx context.Context, args []string) (*oauth2.Token, error) {
@@ -95,7 +122,7 @@ func (c *CliTokenSource) execCliCommand(ctx context.Context, args []string) (*oa
 	if err != nil {
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) {
-			return nil, fmt.Errorf("cannot get access token: %s", strings.TrimSpace(string(exitErr.Stderr)))
+			return nil, fmt.Errorf("cannot get access token: %q", strings.TrimSpace(string(exitErr.Stderr)))
 		}
 		return nil, fmt.Errorf("cannot get access token: %w", err)
 	}
@@ -115,10 +142,13 @@ func (c *CliTokenSource) execCliCommand(ctx context.Context, args []string) (*oa
 }
 
 // isUnknownFlagError returns true if the error indicates the CLI does not
-// recognize the --profile flag. This happens with older CLI versions that
-// predate profile-based token lookup.
-func isUnknownFlagError(err error) bool {
-	return strings.Contains(err.Error(), "unknown flag: --profile")
+// recognize a flag. Pass a specific flag (e.g. "--profile") to check for that
+// flag, or pass "" to match any "unknown flag:" error.
+func isUnknownFlagError(err error, flag string) bool {
+	if flag == "" {
+		return strings.Contains(err.Error(), "unknown flag:")
+	}
+	return strings.Contains(err.Error(), "unknown flag: "+flag)
 }
 
 // parseExpiry parses an expiry time string in multiple formats for cross-SDK compatibility.