Make all prompts write to stderr instead of stdout (#4301)

## Changes

All prompt functions now consistently write to stderr, removing the need
to check if stdout is a TTY for prompt support.

Depends on #4298.

## Why

This simplifies the logic and follows Unix conventions: keep stdout
clean for data/piping, use stderr for interactive UI elements. Now
prompts work even when stdout is redirected (e.g., `databricks cmd | tee
output.txt`).

---------

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: pietern

libs/cmdio/capabilities.go

@@ -37,8 +37,9 @@ func (c Capabilities) SupportsInteractive() bool {
 }
 
 // SupportsPrompt returns true if terminal supports user prompting.
+// Prompts write to stderr and read from stdin, so we only need those to be TTYs.
 func (c Capabilities) SupportsPrompt() bool {
-	return c.SupportsInteractive() && c.stdinIsTTY && c.stdoutIsTTY && !c.isGitBash
+	return c.SupportsInteractive() && c.stdinIsTTY && !c.isGitBash
 }
 
 // SupportsColor returns true if the given writer supports colored output.

libs/cmdio/capabilities_test.go

@@ -84,15 +84,15 @@ func TestCapabilities_SupportsPrompt(t *testing.T) {
 			expected: false,
 		},
 		{
-			name: "stdout not TTY",
+			name: "stdout not TTY (prompts write to stderr, so this is OK)",
 			caps: Capabilities{
 				stdinIsTTY:  true,
 				stdoutIsTTY: false,
 				stderrIsTTY: true,
 				color:       true,
 				isGitBash:   false,
 			},
-			expected: false,
+			expected: true,
 		},
 		{
 			name: "stderr not TTY",

libs/cmdio/compat.go

@@ -115,9 +115,6 @@ func AskSelect(ctx context.Context, question string, choices []string) (string,
 		return "", err
 	}
 
-	// Note: by default this prompt uses os.Stdin and os.Stdout.
-	// This is contrary to the rest of the original progress logger
-	// functions that write to stderr.
 	prompt := promptui.Select{
 		Label:    last,
 		Items:    choices,
@@ -126,6 +123,8 @@ func AskSelect(ctx context.Context, question string, choices []string) (string,
 			Label:    "{{.}}: ",
 			Selected: last + ": {{.}}",
 		},
+		Stdin:  io.NopCloser(c.in),
+		Stdout: nopWriteCloser{c.err},
 	}
 
 	_, ans, err := prompt.Run()

libs/cmdio/io.go

@@ -16,6 +16,15 @@ import (
 // cmdIO is the private instance, that is not supposed to be accessed
 // outside of `cmdio` package. Use the public package-level functions
 // to access the inner state.
+//
+// Stream Architecture:
+//   - in:  stdin for user input (prompts, confirmations)
+//   - out: stdout for data output (JSON, tables, command results)
+//   - err: stderr for interactive UI (prompts, spinners, logs, diagnostics)
+//
+// This separation enables piping stdout while maintaining interactivity:
+//
+//	databricks clusters list --output json | jq  # User sees prompts, jq gets JSON
 type cmdIO struct {
 	capabilities   Capabilities
 	outputFormat   flags.Output
@@ -75,7 +84,8 @@ func (c *cmdIO) Select(items []Tuple, label string) (id string, err error) {
 			Active:   `{{.Name | bold}} ({{.Id|faint}})`,
 			Inactive: `{{.Name}}`,
 		},
-		Stdin: io.NopCloser(c.in),
+		Stdin:  io.NopCloser(c.in),
+		Stdout: nopWriteCloser{c.err},
 	}).Run()
 	if err != nil {
 		return id, err
@@ -110,6 +120,8 @@ func (c *cmdIO) Secret(label string) (value string, err error) {
 		Label:       label,
 		Mask:        '*',
 		HideEntered: true,
+		Stdin:       io.NopCloser(c.in),
+		Stdout:      nopWriteCloser{c.err},
 	})
 
 	return prompt.Run()
@@ -132,7 +144,7 @@ func Prompt(ctx context.Context) *promptui.Prompt {
 	c := fromContext(ctx)
 	return &promptui.Prompt{
 		Stdin:  io.NopCloser(c.in),
-		Stdout: nopWriteCloser{c.out},
+		Stdout: nopWriteCloser{c.err},
 	}
 }