address comments and better

shreyas-goenka · shreyas-goenka · commit c5f44a332ab7 · 2025-10-09T16:43:13.000+02:00
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
@@ -466,6 +466,13 @@ func getSkipReason(config *internal.TestConfig, configPath string) string {
 			return fmt.Sprintf("Disabled via RequiresCluster setting in %s (TEST_DEFAULT_CLUSTER_ID is empty)", configPath)
 		}
 
+		if isTruePtr(config.RequiresWorkspaceFilesystem) {
+			isDBR := os.Getenv("DATABRICKS_RUNTIME_VERSION") != ""
+			if !isDBR || !WorkspaceTmpDir {
+				return fmt.Sprintf("Disabled via RequiresWorkspaceFilesystem setting in %s (DATABRICKS_RUNTIME_VERSION=%s, WorkspaceTmpDir=%v)", configPath, os.Getenv("DATABRICKS_RUNTIME_VERSION"), WorkspaceTmpDir)
+			}
+		}
+
 	} else {
 		// Local run
 		if !isTruePtr(config.Local) {
@@ -516,7 +523,7 @@ func runTest(t *testing.T,
 		// If the test is being run on DBR, auth is already configured
 		// by the dbr_runner notebook by reading a token from the notebook context and
 		// setting DATABRICKS_TOKEN and DATABRICKS_HOST environment variables.
-		_, _, tmpDir = workspaceTmpDir(context.Background(), t)
+		tmpDir = workspaceTmpDir(context.Background(), t)
 
 		// Run DBR tests on the workspace file system to mimic usage from
 		// DABs in the workspace.
diff --git a/acceptance/dbr_test.go b/acceptance/dbr_test.go
@@ -23,7 +23,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func workspaceTmpDir(ctx context.Context, t *testing.T) (*databricks.WorkspaceClient, filer.Filer, string) {
+func workspaceTmpDir(ctx context.Context, t *testing.T) string {
 	w, err := databricks.NewWorkspaceClient()
 	require.NoError(t, err)
 
@@ -38,21 +38,17 @@ func workspaceTmpDir(ctx context.Context, t *testing.T) (*databricks.WorkspaceCl
 		uuid.New().String(),
 	)
 
+	// Create the directory using os.MkdirAll (via FUSE)
+	err = os.MkdirAll(tmpDir, 0o755)
+	require.NoError(t, err)
+
 	t.Cleanup(func() {
-		err := w.Workspace.Delete(ctx, workspace.Delete{
-			Path:      tmpDir,
-			Recursive: true,
-		})
+		// Remove the directory using os.RemoveAll (via FUSE)
+		err := os.RemoveAll(tmpDir)
 		assert.NoError(t, err)
 	})
 
-	err = w.Workspace.MkdirsByPath(ctx, tmpDir)
-	require.NoError(t, err)
-
-	f, err := filer.NewWorkspaceFilesClient(w, tmpDir)
-	require.NoError(t, err)
-
-	return w, f, tmpDir
+	return tmpDir
 }
 
 // Stable scratch directory to run and iterate on DBR tests.
diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go
@@ -55,6 +55,11 @@ type TestConfig struct {
 	// If true and Cloud=true, run this test only if a default warehouse is available in the cloud environment
 	RequiresWarehouse *bool
 
+	// If true run this test only if running on DBR with workspace filesystem
+	// Note that this implicitly implies Cloud=true since running on the workspace
+	// file system is only supported for integration tests.
+	RequiresWorkspaceFilesystem *bool
+
 	// If set, current user will be set to a service principal-like UUID instead of email (default is false)
 	IsServicePrincipal *bool
 
diff --git a/acceptance/internal/materialized_config.go b/acceptance/internal/materialized_config.go
@@ -9,30 +9,32 @@ import (
 const MaterializedConfigFile = "out.test.toml"
 
 type MaterializedConfig struct {
-	GOOS                 map[string]bool     `toml:"GOOS,omitempty"`
-	CloudEnvs            map[string]bool     `toml:"CloudEnvs,omitempty"`
-	Local                *bool               `toml:"Local,omitempty"`
-	Cloud                *bool               `toml:"Cloud,omitempty"`
-	CloudSlow            *bool               `toml:"CloudSlow,omitempty"`
-	RequiresUnityCatalog *bool               `toml:"RequiresUnityCatalog,omitempty"`
-	RequiresCluster      *bool               `toml:"RequiresCluster,omitempty"`
-	RequiresWarehouse    *bool               `toml:"RequiresWarehouse,omitempty"`
-	EnvMatrix            map[string][]string `toml:"EnvMatrix,omitempty"`
+	GOOS                        map[string]bool     `toml:"GOOS,omitempty"`
+	CloudEnvs                   map[string]bool     `toml:"CloudEnvs,omitempty"`
+	Local                       *bool               `toml:"Local,omitempty"`
+	Cloud                       *bool               `toml:"Cloud,omitempty"`
+	CloudSlow                   *bool               `toml:"CloudSlow,omitempty"`
+	RequiresUnityCatalog        *bool               `toml:"RequiresUnityCatalog,omitempty"`
+	RequiresCluster             *bool               `toml:"RequiresCluster,omitempty"`
+	RequiresWarehouse           *bool               `toml:"RequiresWarehouse,omitempty"`
+	RequiresWorkspaceFilesystem *bool               `toml:"RequiresWorkspaceFilesystem,omitempty"`
+	EnvMatrix                   map[string][]string `toml:"EnvMatrix,omitempty"`
 }
 
 // GenerateMaterializedConfig creates a TOML representation of the configuration fields
 // that determine where and how a test is executed
 func GenerateMaterializedConfig(config TestConfig) (string, error) {
 	materialized := MaterializedConfig{
-		GOOS:                 config.GOOS,
-		CloudEnvs:            config.CloudEnvs,
-		Local:                config.Local,
-		Cloud:                config.Cloud,
-		CloudSlow:            config.CloudSlow,
-		RequiresUnityCatalog: config.RequiresUnityCatalog,
-		RequiresCluster:      config.RequiresCluster,
-		RequiresWarehouse:    config.RequiresWarehouse,
-		EnvMatrix:            config.EnvMatrix,
+		GOOS:                        config.GOOS,
+		CloudEnvs:                   config.CloudEnvs,
+		Local:                       config.Local,
+		Cloud:                       config.Cloud,
+		CloudSlow:                   config.CloudSlow,
+		RequiresUnityCatalog:        config.RequiresUnityCatalog,
+		RequiresCluster:             config.RequiresCluster,
+		RequiresWarehouse:           config.RequiresWarehouse,
+		RequiresWorkspaceFilesystem: config.RequiresWorkspaceFilesystem,
+		EnvMatrix:                   config.EnvMatrix,
 	}
 
 	var buf bytes.Buffer
diff --git a/acceptance/scratch_dbr_runner.ipynb b/acceptance/scratch_dbr_runner.ipynb
@@ -10,6 +10,7 @@
     "import tarfile\n",
     "from pathlib import Path\n",
     "\n",
+    "\n",
     "def extract_cli_archive():\n",
     "    src = \"archive.tar.gz\"\n",
     "    dst = Path(\"/tmp/cli_archive\")\n",
@@ -19,6 +20,7 @@
     "\n",
     "    print(f\"Extracted {src} to {dst}\")\n",
     "\n",
+    "\n",
     "extract_cli_archive()"
    ]
   },
@@ -46,6 +48,7 @@
     "import subprocess\n",
     "import multiprocessing\n",
     "\n",
+    "\n",
     "class Runner:\n",
     "    def __init__(self, archive_dir):\n",
     "        # Load environment variables to set in the test runner.\n",
@@ -73,37 +76,84 @@
     "        self.terraform_dir = os.getcwd() + \"/terraform\"\n",
     "\n",
     "    def run(self, prefix):\n",
-    "        cmd = [\"go\", \"tool\", \"gotestsum\", \"--format\", \"testname\", \"--no-summary=skipped\", \"--\",\n",
-    "           \"-timeout\", \"7200s\", \"-test.v\", \"-run\", prefix, \"github.com/databricks/cli/acceptance\",\n",
-    "           \"-workspace-tmp-dir\", \"-terraform-dir\", self.terraform_dir, \"-tail\"]\n",
+    "        cmd = [\n",
+    "            \"go\",\n",
+    "            \"tool\",\n",
+    "            \"gotestsum\",\n",
+    "            \"--format\",\n",
+    "            \"testname\",\n",
+    "            \"--no-summary=skipped\",\n",
+    "            \"--\",\n",
+    "            \"-timeout\",\n",
+    "            \"7200s\",\n",
+    "            \"-test.v\",\n",
+    "            \"-run\",\n",
+    "            prefix,\n",
+    "            \"github.com/databricks/cli/acceptance\",\n",
+    "            \"-workspace-tmp-dir\",\n",
+    "            \"-terraform-dir\",\n",
+    "            self.terraform_dir,\n",
+    "            \"-tail\",\n",
+    "        ]\n",
     "\n",
     "        subprocess.run(cmd, env=self.env, check=True, cwd=self.exec_dir)\n",
     "\n",
-    "   # Debug helper to run commands and see the output.\n",
+    "    # Debug helper to run commands and see the output.\n",
     "    def run_script_with_logs(self, name, input_script):\n",
-    "        cmd = [\"go\", \"tool\", \"gotestsum\", \"--format\", \"testname\", \"--no-summary=skipped\", \"--\",\n",
-    "           \"-timeout\", \"7200s\", \"-test.v\", \"-run\", \"TestAccept/\" + name, \"github.com/databricks/cli/acceptance\",  \"-workspace-tmp-dir\", \"-terraform-dir\", self.terraform_dir, \"-tail\"]\n",
+    "        cmd = [\n",
+    "            \"go\",\n",
+    "            \"tool\",\n",
+    "            \"gotestsum\",\n",
+    "            \"--format\",\n",
+    "            \"testname\",\n",
+    "            \"--no-summary=skipped\",\n",
+    "            \"--\",\n",
+    "            \"-timeout\",\n",
+    "            \"7200s\",\n",
+    "            \"-test.v\",\n",
+    "            \"-run\",\n",
+    "            \"TestAccept/\" + name,\n",
+    "            \"github.com/databricks/cli/acceptance\",\n",
+    "            \"-workspace-tmp-dir\",\n",
+    "            \"-terraform-dir\",\n",
+    "            self.terraform_dir,\n",
+    "            \"-tail\",\n",
+    "        ]\n",
     "\n",
     "        with open(self.exec_dir / \"acceptance\" / name / \"script\", \"w\") as f:\n",
     "            f.write(input_script)\n",
     "\n",
     "        subprocess.run(cmd, env=self.env, check=True, cwd=self.exec_dir)\n",
     "\n",
-    "   # Debug helper to run commands and see the output.\n",
+    "    # Debug helper to run commands and see the output.\n",
     "    def run_with_script(self, name, input_script):\n",
-    "        cmd = [\"go\", \"test\",\n",
-    "           \"-timeout\", \"7200s\", \"-test.v\", \"-run\",\"TestAccept/\" + name, \"github.com/databricks/cli/acceptance\", \"-update\", \"-workspace-tmp-dir\", \"-terraform-dir\", self.terraform_dir]\n",
+    "        cmd = [\n",
+    "            \"go\",\n",
+    "            \"test\",\n",
+    "            \"-timeout\",\n",
+    "            \"7200s\",\n",
+    "            \"-test.v\",\n",
+    "            \"-run\",\n",
+    "            \"TestAccept/\" + name,\n",
+    "            \"github.com/databricks/cli/acceptance\",\n",
+    "            \"-update\",\n",
+    "            \"-workspace-tmp-dir\",\n",
+    "            \"-terraform-dir\",\n",
+    "            self.terraform_dir,\n",
+    "        ]\n",
     "\n",
     "        with open(self.exec_dir / \"acceptance\" / name / \"script\", \"w\") as f:\n",
     "            f.write(input_script)\n",
     "\n",
     "        try:\n",
-    "            subprocess.run(cmd, env=self.env, check=True, cwd=self.exec_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n",
+    "            subprocess.run(\n",
+    "                cmd, env=self.env, check=True, cwd=self.exec_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL\n",
+    "            )\n",
     "        except subprocess.CalledProcessError:\n",
     "            pass\n",
     "\n",
     "        with open(self.exec_dir / \"acceptance\" / name / \"output.txt\", \"r\") as f:\n",
-    "            print(f.read())\n"
+    "            print(f.read())"
    ]
   },
   {
@@ -114,7 +164,9 @@
    "outputs": [],
    "source": [
     "runner = Runner(\"/tmp/cli_archive\")\n",
-    "runner.run_with_script(\"bundle/deploy/dashboard/simple_syncroot\", r\"\"\"\n",
+    "runner.run_with_script(\n",
+    "    \"selftest/dbr\",\n",
+    "    r\"\"\"\n",
     "echo \"========================= START OF SCRIPT =====================\"\n",
     "DASHBOARD_DISPLAY_NAME=\"test bundle-deploy-dashboard $(uuid)\"\n",
     "export DASHBOARD_DISPLAY_NAME\n",
@@ -125,7 +177,8 @@
     "\n",
     "trace cat databricks.yml\n",
     "echo \"========================= END OF SCRIPT =====================\"\n",
-    "\"\"\")"
+    "\"\"\",\n",
+    ")"
    ]
   },
   {
@@ -139,7 +192,7 @@
     "\n",
     "# Example:\n",
     "runner = Runner(\"/tmp/cli_archive\")\n",
-    "runner.run(\"TestAccept/bundle/deploy/dashboard/simple_syncroot\")"
+    "runner.run(\"selftest/dbr\")"
    ]
   }
  ],
diff --git a/acceptance/selftest/dbr/script b/acceptance/selftest/dbr/script
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+printf "=== Verify running on DBR\n"
+if [ -z "$DATABRICKS_RUNTIME_VERSION" ]; then
+  echo "ERROR: DATABRICKS_RUNTIME_VERSION is not set"
+  exit 1
+fi
+echo "DATABRICKS_RUNTIME_VERSION is set: $DATABRICKS_RUNTIME_VERSION"
+
+printf "\n=== Verify current directory is on workspace filesystem\n"
+trace pwd
+
+current_dir=$(pwd)
+if [[ ! "$current_dir" =~ ^/Workspace/Users/ ]]; then
+  echo "ERROR: Current directory is not in /Workspace/Users/"
+  echo "Current directory: $current_dir"
+  exit 1
+fi
+
+echo "SUCCESS: Current directory is in workspace filesystem"
+
+printf "\n=== Verify current user matches directory\n"
+expected_path="/Workspace/Users/$CURRENT_USER_NAME"
+if [[ ! "$current_dir" =~ ^$expected_path ]]; then
+  echo "ERROR: Current directory does not start with $expected_path"
+  echo "Current directory: $current_dir"
+  exit 1
+fi
+
+echo "SUCCESS: Current directory is under $expected_path"
diff --git a/acceptance/selftest/dbr/test.toml b/acceptance/selftest/dbr/test.toml
@@ -0,0 +1,2 @@
+Cloud = true
+RequiresWorkspaceFilesystem = true

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Cloud = true`
	`2`	`+RequiresWorkspaceFilesystem = true`