Generalize CLI token source into progressive command list

Replace the three explicit command fields (forceCmd, profileCmd, hostCmd)
with a []cliCommand list and an activeCommandIndex. Feature flags are
defined statically in cliFeatureFlags and stripped progressively to
build commands for older CLI versions.

Token() now iterates from activeCommandIndex, falling back on unknown
flag errors and caching the working command index for subsequent calls.
Adding future flags (e.g. --scopes) requires only a one-line addition
to the cliFeatureFlags slice.

Signed-off-by: Mihai Mitrea <mihai.mitrea@databricks.com>

Author: mihaimitrea-db

NEXT_CHANGELOG.md

@@ -22,6 +22,7 @@
 
 ### Internal Changes
 
+ * Generalize CLI token source into a progressive command list for forward-compatible flag support.
  * Normalize internal token sources on `auth.TokenSource` for proper context propagation ([#1577](https://github.com/databricks/databricks-sdk-go/pull/1577)).
  * Fix `TestAzureGithubOIDCCredentials` hang caused by missing `HTTPTransport` stub: `EnsureResolved` now calls `resolveHostMetadata`, which makes a real network request when no transport is set ([#1550](https://github.com/databricks/databricks-sdk-go/pull/1550)).
  * Bump golang.org/x/crypto from 0.21.0 to 0.45.0 in /examples/slog ([#1566](https://github.com/databricks/databricks-sdk-go/pull/1566)).

config/cli_token_source.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/databricks/databricks-sdk-go/logger"
@@ -31,50 +32,78 @@ type cliTokenResponse struct {
 	Expiry      string `json:"expiry"`
 }
 
-// CliTokenSource fetches OAuth tokens by shelling out to the Databricks CLI.
-// Commands are tried in order: forceCmd -> profileCmd -> hostCmd, progressively
-// falling back to simpler invocations for older CLI versions.
-type CliTokenSource struct {
-	// forceCmd uses --profile with --force-refresh to bypass the CLI's token cache.
-	// Nil when cfg.Profile is empty (--force-refresh requires --profile support).
-	forceCmd []string
+// cliFeatureFlag defines a CLI feature flag and the warning to log when
+// falling back because the CLI does not support it. Ordered newest-first:
+// commands are built by progressively stripping these flags for older CLIs.
+type cliFeatureFlag struct {
+	flag           string
+	warningMessage string
+}
+
+var cliFeatureFlags = []cliFeatureFlag{
+	{"--force-refresh", "Databricks CLI does not support --force-refresh flag. The CLI's token cache may provide stale tokens. Please upgrade your CLI to the latest version."},
+}
 
-	// profileCmd uses --profile for token lookup. Nil when cfg.Profile is empty.
-	profileCmd []string
+const profileFlagWarning = "Databricks CLI does not support --profile flag. Falling back to --host. Please upgrade your CLI to the latest version."
 
-	// hostCmd uses --host as a fallback for CLIs that predate --profile support.
-	// Nil when cfg.Host is empty.
-	hostCmd []string
+// cliCommand is a single CLI invocation with an associated warning message
+// that is logged when this command fails and we fall back to the next one.
+type cliCommand struct {
+	args           []string
+	warningMessage string
+}
+
+// CliTokenSource fetches OAuth tokens by shelling out to the Databricks CLI.
+// It holds a list of commands ordered from most feature-rich to simplest,
+// falling back progressively for older CLI versions that lack newer flags.
+type CliTokenSource struct {
+	commands           []cliCommand
+	activeCommandIndex atomic.Int32
 }
 
 func NewCliTokenSource(cfg *Config) (*CliTokenSource, error) {
 	cliPath, err := findDatabricksCli(cfg.DatabricksCliPath)
 	if err != nil {
 		return nil, err
 	}
-	forceCmd, profileCmd, hostCmd := buildCliCommands(cliPath, cfg)
-	return &CliTokenSource{forceCmd: forceCmd, profileCmd: profileCmd, hostCmd: hostCmd}, nil
+	return &CliTokenSource{commands: buildCliCommands(cliPath, cfg)}, nil
 }
 
-// buildCliCommands constructs the CLI commands for fetching an auth token.
-// When cfg.Profile is set, three commands are built: a --force-refresh variant,
-// a plain --profile variant, and (when host is available) a --host fallback.
-// When cfg.Profile is empty, only --host is returned — the CLI must support
-// --profile before --force-refresh can be used (monotonic feature assumption).
-func buildCliCommands(cliPath string, cfg *Config) ([]string, []string, []string) {
-	var forceCmd, profileCmd, hostCmd []string
+// buildCliCommands constructs the list of CLI commands for fetching an auth
+// token, ordered from most feature-rich to simplest. When cfg.Profile is set,
+// commands include feature flags from [cliFeatureFlags] (stripped progressively)
+// plus a --host fallback when host is available. When cfg.Profile is empty,
+// only --host is returned — the CLI must support --profile before any feature
+// flags can be used (monotonic feature assumption).
+func buildCliCommands(cliPath string, cfg *Config) []cliCommand {
+	var commands []cliCommand
 	if cfg.Profile != "" {
-		profileCmd = []string{cliPath, "auth", "token", "--profile", cfg.Profile}
-		forceCmd = append(profileCmd, "--force-refresh")
+		baseArgs := []string{cliPath, "auth", "token", "--profile", cfg.Profile}
+		for i := 0; i <= len(cliFeatureFlags); i++ {
+			args := append([]string{}, baseArgs...)
+			for _, f := range cliFeatureFlags[i:] {
+				args = append(args, f.flag)
+			}
+			warning := profileFlagWarning
+			if i < len(cliFeatureFlags) {
+				warning = cliFeatureFlags[i].warningMessage
+			}
+			commands = append(commands, cliCommand{args: args, warningMessage: warning})
+		}
 	}
-	if cfg.Host != "" {
-		hostCmd = buildHostCommand(cliPath, cfg)
+	hostArgs := buildHostCommand(cliPath, cfg)
+	if hostArgs != nil {
+		commands = append(commands, cliCommand{args: hostArgs})
 	}
-	return forceCmd, profileCmd, hostCmd
+	return commands
 }
 
 // buildHostCommand constructs the legacy --host based CLI command.
+// Returns nil when cfg.Host is empty.
 func buildHostCommand(cliPath string, cfg *Config) []string {
+	if cfg.Host == "" {
+		return nil
+	}
 	cmd := []string{cliPath, "auth", "token", "--host", cfg.Host}
 	switch cfg.HostType() {
 	case AccountHost:
@@ -84,36 +113,24 @@ func buildHostCommand(cliPath string, cfg *Config) []string {
 }
 
 // Token fetches an OAuth token by shelling out to the Databricks CLI.
-// Commands are tried in order: forceCmd -> profileCmd -> hostCmd, skipping nil
-// entries. Each command falls through to the next on "unknown flag" errors,
-// logging a warning about the unsupported feature.
+// Commands are tried from [activeCommandIndex] forward, falling back to simpler
+// commands when the CLI reports an unknown flag. Once a command succeeds,
+// [activeCommandIndex] is updated so subsequent calls skip commands known to fail.
 func (c *CliTokenSource) Token(ctx context.Context) (*oauth2.Token, error) {
-	if c.forceCmd != nil {
-		tok, err := c.execCliCommand(ctx, c.forceCmd)
-		if err == nil {
-			return tok, nil
-		}
-		if !isUnknownFlagError(err, "--force-refresh") && !isUnknownFlagError(err, "--profile") {
-			return nil, err
-		}
-		logger.Warnf(ctx, "Databricks CLI does not support --force-refresh flag. The CLI's token cache may provide stale tokens. Please upgrade your CLI to the latest version.")
-	}
-
-	if c.profileCmd != nil {
-		tok, err := c.execCliCommand(ctx, c.profileCmd)
+	for i := int(c.activeCommandIndex.Load()); i < len(c.commands); i++ {
+		cmd := c.commands[i]
+		tok, err := c.execCliCommand(ctx, cmd.args)
 		if err == nil {
+			c.activeCommandIndex.Store(int32(i))
 			return tok, nil
 		}
-		if !isUnknownFlagError(err, "--profile") {
+		lastCommand := i == len(c.commands)-1
+		if lastCommand || !isUnknownFlagError(err, "") {
 			return nil, err
 		}
-		logger.Warnf(ctx, "Databricks CLI does not support --profile flag. Falling back to --host. Please upgrade your CLI to the latest version.")
-	}
-
-	if c.hostCmd == nil {
-		return nil, fmt.Errorf("cannot get access token: no CLI commands available")
+		logger.Warnf(ctx, cmd.warningMessage)
 	}
-	return c.execCliCommand(ctx, c.hostCmd)
+	return nil, fmt.Errorf("cannot get access token: no CLI commands configured")
 }
 
 func (c *CliTokenSource) execCliCommand(ctx context.Context, args []string) (*oauth2.Token, error) {