Add -terraform-dir option to acceptance tests (#3497)

## Changes
This flag makes the download location of the Terraform CLI and providers
configurable instead of defaulting to acceptance/build.

## Why
DBR does not allow creating symlinks between the workspace file system
and the local file system mount on an serverless instance.

So in order to run acceptance tests on workspace file system, the
Databricks terraform provider must also live on the workspace file
system.

## Background

We configure file system mirrors in acceptance tests for Terraform
providers in order to only download the provider once:
```
provider_installation {
    filesystem_mirror {
        path = "/Users/shreyas.goenka/repos/cli/acceptance/build/darwin_arm64/tfplugins"
        include = ["registry.terraform.io/databricks/databricks"]
    }
}
```

Because the DBR CLI repo lives on the local file mount, but the tests
are run on the workspace file system, this file system mirror fails
because it cannot create the necessary symlinks during `terraform init`
(which is called during `bundle deploy`).

Making the Terraform installation directory configurable allows us to
install TF on the workspace file system, bypassing this limitation.

## Tests
Manually verified that:
1. The `install_terraform.py` script respects the—terraform-dir value,
and the Terraform CLI and provider are installed in the correct
directory.
2. The correct environment variables are set when the test is executed:
```
DATABRICKS_TF_CLI_CONFIG_FILE=/Users/shreyas.goenka/repos/cli/somecustomdir/.terraformrc
DATABRICKS_TF_EXEC_PATH=/Users/shreyas.goenka/repos/cli/somecustomdir/terraform
```

Author: shreyas-goenka

acceptance/acceptance_test.go

@@ -47,6 +47,7 @@ var (
 	SkipLocal       bool
 	UseVersion      string
 	WorkspaceTmpDir bool
+	TerraformDir    string
 )
 
 // In order to debug CLI running under acceptance test, search for TestInprocessMode and update
@@ -72,6 +73,11 @@ func init() {
 	// DABs in the workspace runs on the workspace file system. This flags does the same for acceptance tests
 	// to simulate an identical environment.
 	flag.BoolVar(&WorkspaceTmpDir, "workspace-tmp-dir", false, "Run tests on the workspace file system (For DBR testing).")
+
+	// Symlinks from workspace file system to local file mount are not supported on DBR. Terraform implicitly
+	// creates these symlinks when a file_mirror is used for a provider (in .terraformrc). This flag
+	// allows us to download the provider to the workspace file system on DBR enabling DBR integration testing.
+	flag.StringVar(&TerraformDir, "terraform-dir", "", "Directory to download the terraform provider to")
 }
 
 const (
@@ -148,9 +154,16 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	t.Setenv("LC_ALL", "C")
 
 	buildDir := filepath.Join(cwd, "build", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
+	err = os.MkdirAll(buildDir, os.ModePerm)
+	require.NoError(t, err)
+
+	terraformDir := TerraformDir
+	if terraformDir == "" {
+		terraformDir = buildDir
+	}
 
-	// Download terraform and provider and create config; this also creates build directory.
-	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".")
+	// Download terraform and provider and create config.
+	RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", terraformDir}, ".")
 
 	wheelPath := buildDatabricksBundlesWheel(t, buildDir)
 	if wheelPath != "" {
@@ -226,12 +239,12 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		repls.Set(testDefaultWarehouseId, "[TEST_DEFAULT_WAREHOUSE_ID]")
 	}
 
-	terraformrcPath := filepath.Join(buildDir, ".terraformrc")
+	terraformrcPath := filepath.Join(terraformDir, ".terraformrc")
 	t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
 	t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
 	repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]")
 
-	terraformExecPath := filepath.Join(buildDir, "terraform") + exeSuffix
+	terraformExecPath := filepath.Join(terraformDir, "terraform") + exeSuffix
 	t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
 	t.Setenv("TERRAFORM", terraformExecPath)
 	repls.SetPath(terraformExecPath, "[TERRAFORM]")