Add host discovery to login.databricks.com flow (#4829)

## Why

The discovery login flow via `login.databricks.com` (PR #4702) is a
separate code path from the regular `--host` login. After #4809 added
SPOG host detection via `.well-known/databricks-config` to the regular
login path, the discovery flow was missing this behavior. Profiles
created via `login.databricks.com` for SPOG hosts had no `account_id`
and no discovery metadata, which breaks re-authentication (because
`ToOAuthArgument()` needs `account_id` to route to unified OAuth).

## Changes

**Before:** `discoveryLogin()` only used token introspection for
`workspace_id` and deliberately skipped saving `account_id`.

**Now:** After getting the host from `login.databricks.com`,
`discoveryLogin()` calls `runHostDiscovery()` on the discovered host to
populate `account_id`, `workspace_id`, and `DiscoveryURL` from
`.well-known/databricks-config`. Token introspection is kept as a
fallback for hosts where discovery is unavailable (e.g. classic
workspace hosts). `account_id` is now saved to the profile.

**Note:** This is not the full SPOG story for discovery login. Most SPOG
workspaces will need additional handling during the login.databricks.com
flow (e.g. workspace selection after discovery detects a multi-workspace
account). A follow-up PR will address this.

## Test plan

- New test: discovery login with SPOG host (mock
`.well-known/databricks-config`) verifies `account_id` and
`workspace_id` come from discovery
- New test: discovery login where host discovery fails verifies fallback
to introspection
- Updated existing tests to assert `account_id` is now saved
- All `go test ./cmd/auth/... ./libs/auth/...` pass
- `make checks` passes

This pull request was AI-assisted by Isaac.

Author: simonfaltum

acceptance/cmd/auth/login/discovery/out.databrickscfg

@@ -3,7 +3,8 @@
 
 [discovery-test]
 host         = [DATABRICKS_URL]
-workspace_id = 12345
+account_id   = test-account-123
+workspace_id = [NUMID]
 auth_type    = databricks-cli
 
 [__settings__]

acceptance/cmd/auth/login/discovery/test.toml

@@ -4,6 +4,9 @@ Ignore = [
 RecordRequests = true
 
 # Override the introspection endpoint so we can verify it gets called.
+# Host discovery (via .well-known/databricks-config) provides workspace_id.
+# Introspection provides account_id as a fallback since the default
+# discovery handler doesn't return it.
 [[Server]]
 Pattern = "GET /api/2.0/tokens/introspect"
 Response.Body = '''

cmd/auth/login.go

@@ -613,24 +613,33 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 		return discoveryErr("login succeeded but no workspace host was discovered", nil)
 	}
 
-	// Get the token for introspection
+	// Run host metadata discovery on the discovered host to detect SPOG hosts
+	// and populate account_id/workspace_id. This ensures profiles created via
+	// login.databricks.com have the same metadata as profiles created via the
+	// regular --host login path.
+	hostArgs := &auth.AuthArguments{Host: discoveredHost}
+	runHostDiscovery(ctx, hostArgs)
+	accountID := hostArgs.AccountID
+	workspaceID := hostArgs.WorkspaceID
+
+	// Best-effort introspection as a fallback for workspace_id when host
+	// metadata discovery didn't return it (e.g. classic workspace hosts).
 	tok, err := persistentAuth.Token()
 	if err != nil {
 		return fmt.Errorf("retrieving token after login: %w", err)
 	}
 
-	// Best-effort introspection for metadata.
-	var workspaceID string
 	introspection, err := dc.IntrospectToken(ctx, discoveredHost, tok.AccessToken)
 	if err != nil {
 		log.Debugf(ctx, "token introspection failed (non-fatal): %v", err)
 	} else {
-		// TODO: Save introspection.AccountID once the SDKs are ready to use
-		// account_id as part of the profile/cache key. Adding it now would break
-		// existing auth flows that don't expect account_id on workspace profiles.
-		workspaceID = introspection.WorkspaceID
+		if workspaceID == "" {
+			workspaceID = introspection.WorkspaceID
+		}
+		if accountID == "" {
+			accountID = introspection.AccountID
+		}
 
-		// Warn if the detected account_id differs from what's already saved in the profile.
 		if existingProfile != nil && existingProfile.AccountID != "" && introspection.AccountID != "" &&
 			existingProfile.AccountID != introspection.AccountID {
 			log.Warnf(ctx, "detected account ID %q differs from existing profile account ID %q",
@@ -641,10 +650,10 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 	configFile := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
 	clearKeys := oauthLoginClearKeys()
 	// Discovery login always produces a workspace-level profile pointing at the
-	// discovered host. Any previous routing metadata (account_id, workspace_id,
-	// is_unified_host, cluster_id, serverless_compute_id) from a prior login to
-	// a different host type must be cleared so they don't leak into the new
-	// profile. workspace_id is re-added only when introspection succeeds.
+	// discovered host. Any previous routing metadata (is_unified_host,
+	// cluster_id, serverless_compute_id) from a prior login to a different host
+	// type must be cleared so they don't leak into the new profile. account_id
+	// and workspace_id are re-added from discovery/introspection results.
 	clearKeys = append(clearKeys,
 		"account_id",
 		"workspace_id",
@@ -656,6 +665,7 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, profileName string,
 		Profile:     profileName,
 		Host:        discoveredHost,
 		AuthType:    authTypeDatabricksCLI,
+		AccountID:   accountID,
 		WorkspaceID: workspaceID,
 		Scopes:      scopesList,
 		ConfigFile:  configFile,

cmd/auth/login_test.go

@@ -678,11 +678,12 @@ func TestDiscoveryLogin_AccountIDMismatchWarning(t *testing.T) {
 	assert.Contains(t, logBuf.String(), "new-account-id")
 	assert.Contains(t, logBuf.String(), "old-account-id")
 
-	// Verify the profile was saved without account_id (not overwritten).
+	// Account ID from introspection is now saved to the profile.
 	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
 	require.NoError(t, err)
 	require.NotNil(t, savedProfile)
 	assert.Equal(t, "https://workspace.example.com", savedProfile.Host)
+	assert.Equal(t, "new-account-id", savedProfile.AccountID)
 	assert.Equal(t, "12345", savedProfile.WorkspaceID)
 }
 
@@ -816,6 +817,83 @@ func TestDiscoveryLogin_ExplicitScopesOverrideExistingProfile(t *testing.T) {
 	assert.Equal(t, "all-apis", savedProfile.Scopes)
 }
 
+func TestDiscoveryLogin_SPOGHostPopulatesAccountIDFromDiscovery(t *testing.T) {
+	// Start a mock server that returns SPOG discovery metadata.
+	server := newDiscoveryServer(t, map[string]any{
+		"account_id":    "discovered-account",
+		"workspace_id":  "discovered-ws",
+		"oidc_endpoint": "https://spog.example.com/oidc/accounts/discovered-account",
+	})
+
+	tmpDir := t.TempDir()
+	configPath := filepath.Join(tmpDir, ".databrickscfg")
+	err := os.WriteFile(configPath, []byte(""), 0o600)
+	require.NoError(t, err)
+	t.Setenv("DATABRICKS_CONFIG_FILE", configPath)
+
+	oauthArg, err := u2m.NewBasicDiscoveryOAuthArgument("DISCOVERY")
+	require.NoError(t, err)
+	oauthArg.SetDiscoveredHost(server.URL)
+
+	dc := &fakeDiscoveryClient{
+		oauthArg: oauthArg,
+		persistentAuth: &fakeDiscoveryPersistentAuth{
+			token: &oauth2.Token{AccessToken: "test-token"},
+		},
+		// Introspection returns different values to verify discovery takes precedence.
+		introspection: &auth.IntrospectionResult{
+			AccountID:   "introspection-account",
+			WorkspaceID: "introspection-ws",
+		},
+	}
+
+	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
+	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", nil, func(string) error { return nil })
+	require.NoError(t, err)
+
+	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
+	require.NoError(t, err)
+	require.NotNil(t, savedProfile)
+	assert.Equal(t, server.URL, savedProfile.Host)
+	assert.Equal(t, "discovered-account", savedProfile.AccountID, "account_id should come from host discovery")
+	assert.Equal(t, "discovered-ws", savedProfile.WorkspaceID, "workspace_id should come from host discovery")
+}
+
+func TestDiscoveryLogin_IntrospectionFallsBackWhenDiscoveryFails(t *testing.T) {
+	tmpDir := t.TempDir()
+	configPath := filepath.Join(tmpDir, ".databrickscfg")
+	err := os.WriteFile(configPath, []byte(""), 0o600)
+	require.NoError(t, err)
+	t.Setenv("DATABRICKS_CONFIG_FILE", configPath)
+
+	// Use a host that won't respond to .well-known/databricks-config.
+	oauthArg, err := u2m.NewBasicDiscoveryOAuthArgument("DISCOVERY")
+	require.NoError(t, err)
+	oauthArg.SetDiscoveredHost("https://workspace.example.com")
+
+	dc := &fakeDiscoveryClient{
+		oauthArg: oauthArg,
+		persistentAuth: &fakeDiscoveryPersistentAuth{
+			token: &oauth2.Token{AccessToken: "test-token"},
+		},
+		introspection: &auth.IntrospectionResult{
+			AccountID:   "introspection-account",
+			WorkspaceID: "introspection-ws",
+		},
+	}
+
+	ctx, _ := cmdio.NewTestContextWithStdout(t.Context())
+	err = discoveryLogin(ctx, dc, "DISCOVERY", time.Second, "", nil, func(string) error { return nil })
+	require.NoError(t, err)
+
+	savedProfile, err := loadProfileByName(ctx, "DISCOVERY", profile.DefaultProfiler)
+	require.NoError(t, err)
+	require.NotNil(t, savedProfile)
+	assert.Equal(t, "https://workspace.example.com", savedProfile.Host)
+	assert.Equal(t, "introspection-account", savedProfile.AccountID, "account_id should fall back to introspection")
+	assert.Equal(t, "introspection-ws", savedProfile.WorkspaceID, "workspace_id should fall back to introspection")
+}
+
 func TestDiscoveryLogin_ClearsStaleRoutingFieldsFromUnifiedProfile(t *testing.T) {
 	tmpDir := t.TempDir()
 	configPath := filepath.Join(tmpDir, ".databrickscfg")
@@ -911,5 +989,6 @@ auth_type = databricks-cli
 	require.NoError(t, err)
 	require.NotNil(t, savedProfile)
 	assert.Equal(t, "https://new-workspace.example.com", savedProfile.Host)
+	assert.Equal(t, "fresh-account", savedProfile.AccountID, "account_id should be saved from introspection")
 	assert.Equal(t, "222222", savedProfile.WorkspaceID, "workspace_id should be updated to fresh introspection value")
 }