Skip local file validation in config sync (#4648)

## Changes

Skip local file validation in config-remote-sync

## Why

Use case:
* user renames pipeline root folder in UI
* folder path is updated in WSFS
* folder path is updated in the resource object by the frontend code
* <== sync starts here
* when CLI attempts config-sync, it fails with “stat: folder not found
error.”

The issue occurs because we have a validation step that checks for the
existence of local files during the path translation phase. We do need
path translation because config sync needs fully resolved paths to
compare it with fields from remote object, but validation is not
necessary in this case


## Tests

1. acceptance test in config_remote_sync
2. unit test in translate_path.go

Author: ilyakuz-db

acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl

@@ -0,0 +1,26 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-$UNIQUE_NAME
+      root_path: ./pipeline_root
+      libraries:
+        - notebook:
+            path: /Users/{{workspace_user_name}}/notebook
+
+  jobs:
+    my_job:
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: ./src/notebook.py
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+
+targets:
+  default:
+    mode: development

acceptance/bundle/config-remote-sync/validation_errors/out.test.toml

@@ -0,0 +1,60 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Set correct paths and add git info remotely
+=== Break local paths to simulate stale config
+=== Sync with broken local paths
+Detected changes in 2 resource(s):
+
+Resource: resources.jobs.my_job
+  git_source: add
+  tasks[task_key='main'].notebook_task.notebook_path: replace
+
+Resource: resources.pipelines.my_pipeline
+  root_path: replace
+
+
+
+=== Configuration changes
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -6,5 +6,5 @@
+     my_pipeline:
+       name: test-pipeline-[UNIQUE_NAME]
+-      root_path: ./pipeline_root
++      root_path: ./pipeline_root_v2
+       libraries:
+         - notebook:
+@@ -16,9 +16,13 @@
+         - task_key: main
+           notebook_task:
+-            notebook_path: ./src/notebook.py
++            notebook_path: /Users/[USERNAME]/notebook
+           new_cluster:
+             spark_version: 13.3.x-snapshot-scala2.12
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1
++      git_source:
++        git_branch: main
++        git_provider: gitHub
++        git_url: https://github.com/databricks/databricks-sdk-go.git
+
+ targets:
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.my_job
+  delete resources.pipelines.my_pipeline
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them:
+  delete resources.pipelines.my_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/validation_errors/output.txt

@@ -0,0 +1,48 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# Create valid paths so that initial deploy succeeds
+mkdir -p pipeline_root src
+echo '# Databricks notebook source' > src/notebook.py
+
+cleanup() {
+  # Restore valid paths for destroy to work (includes pipeline_root_v2 from sync)
+  mkdir -p pipeline_root pipeline_root_v2 src
+  echo '# Databricks notebook source' > src/notebook.py
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+pipeline_id="$(read_id.py my_pipeline)"
+
+
+title "Set correct paths and add git info remotely"
+edit_resource.py pipelines $pipeline_id <<EOF
+r["root_path"] = "${PWD}/pipeline_root_v2"
+EOF
+
+edit_resource.py jobs $job_id <<EOF
+r["tasks"][0]["notebook_task"]["notebook_path"] = "/Users/${CURRENT_USER_NAME}/notebook"
+r["git_source"] = {
+    "git_url": "https://github.com/databricks/databricks-sdk-go.git",
+    "git_branch": "main",
+    "git_provider": "gitHub"
+}
+EOF
+
+
+title "Break local paths to simulate stale config"
+# Remove the directories so the config now references non-existent relative paths
+rm -rf pipeline_root src
+
+
+title "Sync with broken local paths"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+
+title "Configuration changes"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/validation_errors/script

@@ -0,0 +1,8 @@
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup", "src", "pipeline_root"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]

acceptance/bundle/config-remote-sync/validation_errors/test.toml

@@ -149,6 +149,14 @@ type Bundle struct {
 	// files
 	AutoApprove bool
 
+	// SkipLocalFileValidation makes path translation tolerant of missing local files.
+	// When set, TranslatePaths computes workspace paths without verifying files exist.
+	// Used by config-remote-sync: a user may modify resource paths remotely (e.g.,
+	// rename a pipeline root folder in the UI), and the updated paths may not exist
+	// locally. Path translation is still needed to produce fully resolved paths for
+	// comparison with remote state, but local file validation would incorrectly fail.
+	SkipLocalFileValidation bool
+
 	// Tagging is used to normalize tag keys and values.
 	// The implementation depends on the cloud being targeted.
 	Tagging tags.Cloud

bundle/bundle.go

@@ -84,6 +84,14 @@ type translateContext struct {
 	// It is equal to ${workspace.file_path} for regular deployments.
 	// It points to the source root path for source-linked deployments.
 	remoteRoot string
+
+	// skipLocalFileValidation makes path translation tolerant of missing local files.
+	// When set, paths are translated without verifying files exist on the local filesystem.
+	// This is used by config-remote-sync: a user may rename a resource's root folder
+	// in the workspace UI, and the updated path may not exist locally. Path translation
+	// is still needed to produce fully resolved paths for comparison with remote state,
+	// but local file validation would incorrectly fail.
+	skipLocalFileValidation bool
 }
 
 // rewritePath converts a given relative path from the loaded config to a new path based on the passed rewriting function
@@ -178,6 +186,11 @@ func (t *translateContext) rewritePath(
 }
 
 func (t *translateContext) translateNotebookPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
+	if t.skipLocalFileValidation {
+		localRelPathNoExt := strings.TrimSuffix(localRelPath, path.Ext(localRelPath))
+		return path.Join(t.remoteRoot, localRelPathNoExt), nil
+	}
+
 	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		if path.Ext(localFullPath) != notebook.ExtensionNone {
@@ -213,6 +226,10 @@ to contain one of the following file extensions: [%s]`, literal, strings.Join(no
 }
 
 func (t *translateContext) translateFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
+	if t.skipLocalFileValidation {
+		return path.Join(t.remoteRoot, localRelPath), nil
+	}
+
 	nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("file %s not found", literal)
@@ -227,6 +244,10 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local
 }
 
 func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
+	if t.skipLocalFileValidation {
+		return path.Join(t.remoteRoot, localRelPath), nil
+	}
+
 	info, err := t.b.SyncRoot.Stat(localRelPath)
 	if err != nil {
 		return "", err
@@ -242,6 +263,10 @@ func (t *translateContext) translateGlobPath(ctx context.Context, literal, local
 }
 
 func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) {
+	if t.skipLocalFileValidation {
+		return localFullPath, nil
+	}
+
 	info, err := os.Stat(filepath.FromSlash(localFullPath))
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("directory %s not found", literal)
@@ -311,8 +336,9 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex
 
 func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	t := &translateContext{
-		b:    b,
-		seen: make(map[string]string),
+		b:                       b,
+		seen:                    make(map[string]string),
+		skipLocalFileValidation: b.SkipLocalFileValidation,
 	}
 
 	return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){
@@ -327,8 +353,9 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
 
 func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	t := &translateContext{
-		b:    b,
-		seen: make(map[string]string),
+		b:                       b,
+		seen:                    make(map[string]string),
+		skipLocalFileValidation: b.SkipLocalFileValidation,
 	}
 
 	return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){

bundle/config/mutator/translate_paths.go

@@ -1015,3 +1015,101 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
 		b.Config.Resources.Pipelines["pipeline"].Libraries[1].Notebook.Path,
 	)
 }
+
+func TestTranslatePathsWithSkipLocalFileValidation(t *testing.T) {
+	dir := t.TempDir()
+	// Intentionally do NOT create any files — paths are stale/missing.
+
+	b := &bundle.Bundle{
+		SyncRootPath:            dir,
+		BundleRootPath:          dir,
+		SyncRoot:                vfs.MustNew(dir),
+		SkipLocalFileValidation: true,
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath: "/bundle",
+			},
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job": {
+						JobSettings: jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "./src/notebook.py",
+									},
+								},
+								{
+									SparkPythonTask: &jobs.SparkPythonTask{
+										PythonFile: "./src/main.py",
+									},
+								},
+							},
+						},
+					},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					"pipeline": {
+						CreatePipeline: pipelines.CreatePipeline{
+							Libraries: []pipelines.PipelineLibrary{
+								{
+									Notebook: &pipelines.NotebookLibrary{
+										Path: "./src/pipeline_notebook.py",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}})
+
+	diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// Notebook path should be translated (extension stripped) even though file doesn't exist.
+	assert.Equal(t, "/bundle/src/notebook", b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath)
+
+	// File path should be translated even though file doesn't exist.
+	assert.Equal(t, "/bundle/src/main.py", b.Config.Resources.Jobs["job"].Tasks[1].SparkPythonTask.PythonFile)
+
+	// Pipeline notebook path should be translated even though file doesn't exist.
+	assert.Equal(t, "/bundle/src/pipeline_notebook", b.Config.Resources.Pipelines["pipeline"].Libraries[0].Notebook.Path)
+}
+
+func TestTranslatePathsWithSkipLocalFileValidationDirectory(t *testing.T) {
+	dir := t.TempDir()
+	// Intentionally do NOT create pipeline_root directory.
+
+	b := &bundle.Bundle{
+		SyncRootPath:            dir,
+		BundleRootPath:          dir,
+		SyncRoot:                vfs.MustNew(dir),
+		SkipLocalFileValidation: true,
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath: "/bundle",
+			},
+			Resources: config.Resources{
+				Pipelines: map[string]*resources.Pipeline{
+					"pipeline": {
+						CreatePipeline: pipelines.CreatePipeline{
+							RootPath: "./pipeline_root",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}})
+
+	diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// Directory path should be translated even though directory doesn't exist.
+	assert.Equal(t, "/bundle/pipeline_root", b.Config.Resources.Pipelines["pipeline"].RootPath)
+}

bundle/config/mutator/translate_paths_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/configsync"
 	"github.com/databricks/cli/cmd/bundle/utils"
 	"github.com/databricks/cli/cmd/root"
@@ -46,6 +47,9 @@ Examples:
 			ReadState:  true,
 			Build:      true,
 			AlwaysPull: true,
+			InitFunc: func(b *bundle.Bundle) {
+				b.SkipLocalFileValidation = true
+			},
 		})
 		if err != nil {
 			return err