Use filter option with service principal API when lookup variable used (#4913)

## Changes
Use filter option with service principal API when lookup variable used

## Why
Previously we were listing all SPs in the workspace which can be very
slow. Using filter option in API allows to filter out the result
quicker/

## Tests
Updated unit test

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

---------

Co-authored-by: simon <4305831+simonfaltum@users.noreply.github.com>

Author: andrewnester

acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt

@@ -1,4 +1,4 @@
-Error: failed to resolve service-principal: TIDALDBServAccount - usdev, err: ServicePrincipal named 'TIDALDBServAccount - usdev' does not exist
+Error: failed to resolve service-principal: TIDALDBServAccount - usdev, err: service principal named "TIDALDBServAccount - usdev" does not exist
 
 Name: issue-3039
 Target: personal

acceptance/bundle/variables/lookup/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle
+
+variables:
+  sp:
+    lookup:
+      service_principal: "deco-test-spn"
+
+  result:
+    default: ${var.sp}

acceptance/bundle/variables/lookup/out.test.toml

@@ -0,0 +1,14 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "result": {
+    "default": "[UUID]",
+    "value": "[UUID]"
+  },
+  "sp": {
+    "lookup": {
+      "service_principal": "deco-test-spn"
+    },
+    "value": "[UUID]"
+  }
+}

acceptance/bundle/variables/lookup/output.txt

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq '.variables'

acceptance/bundle/variables/lookup/script

@@ -0,0 +1,13 @@
+Local = true
+Cloud = true
+
+[[Server]]
+Pattern = "GET /api/2.0/preview/scim/v2/ServicePrincipals"
+Response.Body = '''{
+  "Resources": [
+    {
+      "displayName": "deco-test-spn",
+      "applicationId": "123e4567-e89b-12d3-a456-426614174000"
+    }
+  ]
+}'''

acceptance/bundle/variables/lookup/test.toml

@@ -1,6 +1,7 @@
 package mutator
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -11,6 +12,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/listing"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 )
@@ -131,11 +133,18 @@ func TestResolveServicePrincipal(t *testing.T) {
 
 	m := mocks.NewMockWorkspaceClient(t)
 	b.SetWorkpaceClient(m.WorkspaceClient)
-	spApi := m.GetMockServicePrincipalsAPI()
-	spApi.EXPECT().GetByDisplayName(mock.Anything, spName).Return(&iam.ServicePrincipal{
-		Id:            "1234",
-		ApplicationId: "app-1234",
-	}, nil)
+
+	api := m.GetMockServicePrincipalsV2API()
+	iterator := listing.SliceIterator[iam.ServicePrincipal]([]iam.ServicePrincipal{
+		{
+			ApplicationId: "app-1234",
+		},
+	})
+	api.EXPECT().
+		List(mock.Anything, iam.ListServicePrincipalsRequest{
+			Filter: fmt.Sprintf("displayName eq '%s'", spName),
+		}).
+		Return(&iterator)
 
 	diags := bundle.Apply(t.Context(), b, ResolveLookupVariables())
 	require.NoError(t, diags.Error())

bundle/config/mutator/resolve_lookup_variables_test.go

@@ -2,8 +2,11 @@ package variable
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/listing"
+	"github.com/databricks/databricks-sdk-go/service/iam"
 )
 
 type resolveServicePrincipal struct {
@@ -12,11 +15,23 @@ type resolveServicePrincipal struct {
 
 func (l resolveServicePrincipal) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
 	//nolint:staticcheck // this API is deprecated but we still need use it as there is no replacement yet.
-	entity, err := w.ServicePrincipals.GetByDisplayName(ctx, l.name)
+	it := w.ServicePrincipalsV2.List(ctx, iam.ListServicePrincipalsRequest{
+		Filter: fmt.Sprintf("displayName eq '%s'", l.name),
+	})
+
+	servicePrincipals, err := listing.ToSliceN(ctx, it, 1)
 	if err != nil {
 		return "", err
 	}
-	return entity.ApplicationId, nil
+	if len(servicePrincipals) == 0 {
+		return "", fmt.Errorf("service principal named %q does not exist", l.name)
+	}
+
+	if len(servicePrincipals) > 1 {
+		return "", fmt.Errorf("multiple service principals found with display name %q", l.name)
+	}
+
+	return servicePrincipals[0].ApplicationId, nil
 }
 
 func (l resolveServicePrincipal) String() string {

bundle/config/variable/resolve_service_principal.go

@@ -3,8 +3,8 @@ package variable
 import (
 	"testing"
 
-	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/listing"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -14,12 +14,17 @@ import (
 func TestResolveServicePrincipal_ResolveSuccess(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
 
-	api := m.GetMockServicePrincipalsAPI()
-	api.EXPECT().
-		GetByDisplayName(mock.Anything, "service-principal").
-		Return(&iam.ServicePrincipal{
+	api := m.GetMockServicePrincipalsV2API()
+	iterator := listing.SliceIterator[iam.ServicePrincipal]([]iam.ServicePrincipal{
+		{
 			ApplicationId: "5678",
-		}, nil)
+		},
+	})
+	api.EXPECT().
+		List(mock.Anything, iam.ListServicePrincipalsRequest{
+			Filter: "displayName eq 'service-principal'",
+		}).
+		Return(&iterator)
 
 	ctx := t.Context()
 	l := resolveServicePrincipal{name: "service-principal"}
@@ -31,15 +36,18 @@ func TestResolveServicePrincipal_ResolveSuccess(t *testing.T) {
 func TestResolveServicePrincipal_ResolveNotFound(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
 
-	api := m.GetMockServicePrincipalsAPI()
+	api := m.GetMockServicePrincipalsV2API()
+	iterator := listing.SliceIterator[iam.ServicePrincipal]([]iam.ServicePrincipal{})
 	api.EXPECT().
-		GetByDisplayName(mock.Anything, "service-principal").
-		Return(nil, &apierr.APIError{StatusCode: 404})
+		List(mock.Anything, iam.ListServicePrincipalsRequest{
+			Filter: "displayName eq 'service-principal'",
+		}).
+		Return(&iterator)
 
 	ctx := t.Context()
 	l := resolveServicePrincipal{name: "service-principal"}
 	_, err := l.Resolve(ctx, m.WorkspaceClient)
-	require.ErrorIs(t, err, apierr.ErrNotFound)
+	require.ErrorContains(t, err, "service principal named \"service-principal\" does not exist")
 }
 
 func TestResolveServicePrincipal_String(t *testing.T) {