Remove deprecated PyDABs support (#3102)

## Changes
Remove deprecated `experimental/pydabs` support.

Leave a minimal amount of code to generate errors if deprecated
properties are used. Otherwise, it's possible that resources are going
to be unexpectedly deleted.

## Why
`experimental/pydabs` was replaced with `experimental/python`

Author: kanterov

NEXT_CHANGELOG.md

@@ -9,5 +9,7 @@
 ### CLI
 
 ### Bundles
+* Remove support for deprecated `experimental/pydabs` config, use `experimental/python` instead. See [Configuration in Python
+](https://docs.databricks.com/dev-tools/bundles/python). ([#3102](https://github.com/databricks/cli/pull/3102))
 
 ### API Changes

acceptance/bundle/debug/out.stderr.txt

@@ -45,8 +45,6 @@
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources)
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=TranslatePathsDashboards
-10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load)
-10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(init)
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources)
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(apply_mutators)
 10:07:59 Debug: Apply pid=12345 mutator=CheckPermissions

bundle/config/experimental.go

@@ -24,9 +24,6 @@ type Experimental struct {
 	// as a top-level permission in the DAB.
 	UseLegacyRunAs bool `json:"use_legacy_run_as,omitempty"`
 
-	// PyDABs determines whether to load the 'databricks-pydabs' package.
-	//
-	// PyDABs allows to define bundle configuration using Python.
 	// PyDABs is deprecated use Python instead.
 	PyDABs PyDABs `json:"pydabs,omitempty"`
 
@@ -70,18 +67,6 @@ type Python struct {
 type PyDABs struct {
 	// Enabled is a flag to enable the feature.
 	Enabled bool `json:"enabled,omitempty"`
-
-	// VEnvPath is path to the virtual environment.
-	//
-	// If enabled, PyDABs will execute code within this environment. If disabled,
-	// it defaults to using the Python interpreter available in the current shell.
-	VEnvPath string `json:"venv_path,omitempty"`
-
-	// Import contains a list Python packages with PyDABs code.
-	//
-	// These packages are imported to discover resources, resource generators, and mutators.
-	// This list can include namespace packages, which causes the import of nested packages.
-	Import []string `json:"import,omitempty"`
 }
 
 type (

bundle/config/mutator/python/python_mutator.go

@@ -36,39 +36,6 @@ import (
 type phase string
 
 const (
-	// PythonMutatorPhaseLoad is the phase in which bundle configuration is loaded.
-	//
-	// At this stage, PyDABs adds statically defined resources to the bundle configuration.
-	// Which resources are added should be deterministic and not depend on the bundle configuration.
-	//
-	// We also open for possibility of appending other sections of bundle configuration,
-	// for example, adding new variables. However, this is not supported yet, and CLI rejects
-	// such changes.
-	//
-	// Deprecated, left for backward-compatibility with PyDABs.
-	PythonMutatorPhaseLoad phase = "load"
-
-	// PythonMutatorPhaseInit is the phase after bundle configuration was loaded, and
-	// the list of statically declared resources is known.
-	//
-	// At this stage, PyDABs adds resources defined using generators, or mutates existing resources,
-	// including the ones defined using YAML.
-	//
-	// During this process, within generator and mutators, PyDABs can access:
-	// - selected deployment target
-	// - bundle variables values
-	// - variables provided through CLI arguments or environment variables
-	//
-	// The following is not available:
-	// - variables referencing other variables are in unresolved format
-	//
-	// PyDABs can output YAML containing references to variables, and CLI should resolve them.
-	//
-	// Existing resources can't be removed, and CLI rejects such changes.
-	//
-	// Deprecated, left for backward-compatibility with PyDABs.
-	PythonMutatorPhaseInit phase = "init"
-
 	// PythonMutatorPhaseLoadResources is the phase in which YAML configuration was loaded.
 	//
 	// At this stage, we execute Python code to load resources defined in Python.
@@ -150,24 +117,11 @@ func getOpts(b *bundle.Bundle, phase phase) (opts, error) {
 	pydabsEnabled := !reflect.DeepEqual(experimental.PyDABs, config.PyDABs{})
 	pythonEnabled := !reflect.DeepEqual(experimental.Python, config.Python{})
 
-	if pydabsEnabled && pythonEnabled {
-		return opts{}, errors.New("both experimental/pydabs and experimental/python are enabled, only one can be enabled")
-	} else if pydabsEnabled {
-		if !experimental.PyDABs.Enabled {
-			return opts{}, nil
-		}
+	if pydabsEnabled {
+		return opts{}, errors.New("experimental/pydabs is deprecated, use experimental/python instead (https://docs.databricks.com/dev-tools/bundles/python)")
+	}
 
-		// don't execute for phases for 'python' section
-		if phase == PythonMutatorPhaseInit || phase == PythonMutatorPhaseLoad {
-			return opts{
-				enabled:       true,
-				venvPath:      experimental.PyDABs.VEnvPath,
-				loadLocations: false, // not supported in PyDABs
-			}, nil
-		} else {
-			return opts{}, nil
-		}
-	} else if pythonEnabled {
+	if pythonEnabled {
 		// don't execute for phases for 'pydabs' section
 		if phase == PythonMutatorPhaseLoadResources || phase == PythonMutatorPhaseApplyMutators {
 			return opts{
@@ -258,14 +212,9 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 		return newRoot, nil
 	})
 
-	// don't instrument deprecated phases
-	if m.phase == PythonMutatorPhaseLoadResources || m.phase == PythonMutatorPhaseApplyMutators {
-		// we can precisely track resources that are added/updated, so sum doesn't double-count
-		b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size())
-		b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size())
-	} else {
-		mutateDiags = mutateDiags.Extend(diag.Warningf("experimental/pydabs is deprecated and will be removed in future versions, use experimental/python instead"))
-	}
+	// we can precisely track resources that are added/updated, so sum doesn't double-count
+	b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size())
+	b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size())
 
 	if err == mutateDiagsHasError {
 		if !mutateDiags.HasError() {

bundle/config/mutator/python/python_mutator_test.go

@@ -24,18 +24,6 @@ import (
 	"github.com/databricks/cli/libs/process"
 )
 
-func TestPythonMutator_Name_load(t *testing.T) {
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
-
-	assert.Equal(t, "PythonMutator(load)", mutator.Name())
-}
-
-func TestPythonMutator_Name_init(t *testing.T) {
-	mutator := PythonMutator(PythonMutatorPhaseInit)
-
-	assert.Equal(t, "PythonMutator(init)", mutator.Name())
-}
-
 func TestPythonMutator_Name_loadResources(t *testing.T) {
 	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 
@@ -293,7 +281,7 @@ func TestPythonMutator_disabled(t *testing.T) {
 	b := loadYaml("databricks.yml", ``)
 
 	ctx := context.Background()
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.NoError(t, diag.Error())
@@ -334,19 +322,17 @@ func TestGetOps_Python(t *testing.T) {
 }
 
 func TestGetOps_PyDABs(t *testing.T) {
-	actual, err := getOpts(&bundle.Bundle{
+	_, err := getOpts(&bundle.Bundle{
 		Config: config.Root{
 			Experimental: &config.Experimental{
 				PyDABs: config.PyDABs{
-					VEnvPath: ".venv",
-					Enabled:  true,
+					Enabled: true,
 				},
 			},
 		},
-	}, PythonMutatorPhaseInit)
+	}, PythonMutatorPhaseLoadResources)
 
-	assert.NoError(t, err)
-	assert.Equal(t, opts{venvPath: ".venv", enabled: true, loadLocations: false}, actual)
+	assert.Error(t, err, "experimental/pydabs is deprecated, use experimental/python instead (https://docs.databricks.com/dev-tools/bundles/python)")
 }
 
 func TestGetOps_empty(t *testing.T) {

bundle/deploy/terraform/init.go

@@ -269,8 +269,6 @@ func setUserAgentExtraEnvVar(environ map[string]string, b *bundle.Bundle) error
 
 		if hasPython {
 			products = append(products, "databricks-pydabs/0.7.0")
-		} else if experimental.PyDABs.Enabled {
-			products = append(products, "databricks-pydabs/0.0.0")
 		}
 	}
 

bundle/deploy/terraform/init_test.go

@@ -250,26 +250,6 @@ func TestSetProxyEnvVars(t *testing.T) {
 	assert.ElementsMatch(t, []string{"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"}, maps.Keys(env))
 }
 
-func TestSetUserAgentExtra_PyDABs(t *testing.T) {
-	b := &bundle.Bundle{
-		BundleRootPath: t.TempDir(),
-		Config: config.Root{
-			Experimental: &config.Experimental{
-				PyDABs: config.PyDABs{
-					Enabled: true,
-				},
-			},
-		},
-	}
-
-	env := make(map[string]string, 0)
-	err := setUserAgentExtraEnvVar(env, b)
-	require.NoError(t, err)
-	assert.Equal(t, map[string]string{
-		"DATABRICKS_USER_AGENT_EXTRA": "cli/0.0.0-dev databricks-pydabs/0.0.0",
-	}, env)
-}
-
 func TestSetUserAgentExtra_Python(t *testing.T) {
 	b := &bundle.Bundle{
 		BundleRootPath: t.TempDir(),

bundle/phases/initialize.go

@@ -130,8 +130,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		// Static resources (e.g. YAML) are already loaded, we initialize and normalize them before Python
 		resourcemutator.ProcessStaticResources(),
 
-		pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
-		pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
 		pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoadResources),
 		pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators),
 		// This is the last mutator that can change bundle resources.