acc: Added materialised test config as out.config.json golden file (#3174)

## Changes
Added materialised test config as `out.config.toml` golden file.
Now each tests compares the test config it has (defined in test.toml and
nested folders) with the config stored in `out.config.toml` which is
materialised as golden file.

## Why
This gives us transparency and visibility into what are the settings
each tests is run against. Any changes to the parent test settings won't
go unnoticed due to golden file comparison.

## Tests
Existing tests succeed

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: andrewnester

acceptance/acceptance_test.go

@@ -73,6 +73,8 @@ const (
 	MaxFileSize      = 100_000
 	// Filename to save replacements to (used by diff.py)
 	ReplsFile = "repls.json"
+	// Filename for materialized config (used as golden file)
+	MaterializedConfigFile = "out.test.toml"
 
 	// ENVFILTER allows filtering subtests matching certain env var.
 	// e.g. ENVFILTER=SERVERLESS=yes will run all tests that run SERVERLESS to "yes"
@@ -464,11 +466,19 @@ func runTest(t *testing.T,
 	scriptContents := readMergedScriptContents(t, dir)
 	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
 
+	// Generate materialized config for this test
+	materializedConfig, err := internal.GenerateMaterializedConfig(config)
+	require.NoError(t, err)
+	testutil.WriteFile(t, filepath.Join(tmpDir, internal.MaterializedConfigFile), materializedConfig)
+
 	inputs := make(map[string]bool, 2)
 	outputs := make(map[string]bool, 2)
 	err = CopyDir(dir, tmpDir, inputs, outputs)
 	require.NoError(t, err)
 
+	// Add materialized config to outputs for comparison
+	outputs[internal.MaterializedConfigFile] = true
+
 	bundleConfigTarget := "databricks.yml"
 	if config.BundleConfigTarget != nil {
 		bundleConfigTarget = *config.BundleConfigTarget
@@ -600,7 +610,11 @@ func runTest(t *testing.T,
 
 	// Compare expected outputs
 	for relPath := range outputs {
-		doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
+		skipRepls := false
+		if relPath == internal.MaterializedConfigFile {
+			skipRepls = true
+		}
+		doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, skipRepls)
 	}
 
 	// Make sure there are not unaccounted for new files
@@ -623,7 +637,7 @@ func runTest(t *testing.T,
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
-			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
+			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false)
 		}
 	}
 
@@ -662,7 +676,7 @@ func addEnvVar(t *testing.T, env []string, repls *testdiff.ReplacementsContext,
 	return append(env, key+"="+newValue)
 }
 
-func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
+func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool, skipRepls bool) {
 	pathRef := filepath.Join(dirRef, relPath)
 	pathNew := filepath.Join(dirNew, relPath)
 	bufRef, okRef := tryReading(t, pathRef)
@@ -677,7 +691,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// Apply replacements to the new value only.
 	// The reference value is stored after applying replacements.
-	if !NoRepl {
+	if !NoRepl && !skipRepls {
 		valueNew = repls.Replace(valueNew)
 	}
 

acceptance/auth/bundle_and_profile/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/auth/credentials/basic/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/auth/credentials/oauth/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/auth/credentials/pat/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/apps/app_yaml/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/apps/config_section/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml

@@ -0,0 +1,6 @@
+Local = false
+Cloud = true
+RequiresUnityCatalog = true
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml

@@ -0,0 +1,6 @@
+Local = false
+Cloud = true
+RequiresUnityCatalog = true
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]