Sort job tasks by task key (#3212)

denik · web-flow · commit 3812c9a39264 · 2025-07-09T17:37:36.000+02:00
## Changes Sort job tasks by task key. ## Why This is what terraform provider does before deployment. Doing so earlier exposes this changes to the user in bundle validate and make it possible to match requests done by TF and direct backends. https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343 ## Tests New tests enabled on direct backend.
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -14,5 +14,6 @@
 
 ### Bundles
 * Fix default search location for whl artifacts ([#3184](https://github.com/databricks/cli/pull/3184)). This was a regression introduced in 0.255.0.
+* The job tasks are now sorted by task key in "bundle validate/summary" output ([#3212](https://github.com/databricks/cli/pull/3212))
 
 ### API Changes
diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml
@@ -2,4 +2,4 @@ Local = true
 Cloud = false
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/artifacts/whl_dynamic/test.toml b/acceptance/bundle/artifacts/whl_dynamic/test.toml
@@ -1,5 +1,3 @@
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key like terraform does
-
 [[Repls]]
 Old = '\\\\'
 New = '/'
diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml
@@ -2,4 +2,4 @@ Local = true
 Cloud = false
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/test.toml
@@ -1,2 +1 @@
 BundleConfig.default_name = ""
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key
diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json
@@ -1,18 +1,4 @@
 [
-  {
-    "job_cluster_key": "default",
-    "notebook_task": {
-      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook"
-    },
-    "task_key": "notebook_example"
-  },
-  {
-    "job_cluster_key": "default",
-    "spark_python_task": {
-      "python_file": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file.py"
-    },
-    "task_key": "spark_python_example"
-  },
   {
     "dbt_task": {
       "commands": [
@@ -26,13 +12,10 @@
   },
   {
     "job_cluster_key": "default",
-    "sql_task": {
-      "file": {
-        "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/sql.sql"
-      },
-      "warehouse_id": "cafef00d"
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook"
     },
-    "task_key": "sql_example"
+    "task_key": "notebook_example"
   },
   {
     "job_cluster_key": "default",
@@ -63,5 +46,22 @@
       "main_class_name": "com.example.Main"
     },
     "task_key": "spark_jar_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
+    },
+    "task_key": "sql_example"
   }
 ]
diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json
@@ -1,18 +1,4 @@
 [
-  {
-    "job_cluster_key": "default",
-    "notebook_task": {
-      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
-    },
-    "task_key": "notebook_example"
-  },
-  {
-    "job_cluster_key": "default",
-    "spark_python_task": {
-      "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
-    },
-    "task_key": "spark_python_example"
-  },
   {
     "dbt_task": {
       "commands": [
@@ -25,14 +11,33 @@
     "task_key": "dbt_example"
   },
   {
+    "for_each_task": {
+      "task": {
+        "notebook_task": {
+          "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
+        }
+      }
+    },
     "job_cluster_key": "default",
-    "sql_task": {
-      "file": {
-        "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/sql.sql"
-      },
-      "warehouse_id": "cafef00d"
+    "task_key": "for_each_notebook_example"
+  },
+  {
+    "for_each_task": {
+      "task": {
+        "job_cluster_key": "default",
+        "spark_python_task": {
+          "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
+        }
+      }
     },
-    "task_key": "sql_example"
+    "task_key": "for_each_spark_python_example"
+  },
+  {
+    "job_cluster_key": "default",
+    "notebook_task": {
+      "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
+    },
+    "task_key": "notebook_example"
   },
   {
     "job_cluster_key": "default",
@@ -65,25 +70,20 @@
     "task_key": "spark_jar_example"
   },
   {
-    "for_each_task": {
-      "task": {
-        "notebook_task": {
-          "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook"
-        }
-      }
-    },
     "job_cluster_key": "default",
-    "task_key": "for_each_notebook_example"
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
+    },
+    "task_key": "spark_python_example"
   },
   {
-    "for_each_task": {
-      "task": {
-        "job_cluster_key": "default",
-        "spark_python_task": {
-          "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py"
-        }
-      }
+    "job_cluster_key": "default",
+    "sql_task": {
+      "file": {
+        "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/sql.sql"
+      },
+      "warehouse_id": "cafef00d"
     },
-    "task_key": "for_each_spark_python_example"
+    "task_key": "sql_example"
   }
 ]
diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json
@@ -75,24 +75,6 @@
           "dev": "[USERNAME]"
         },
         "tasks": [
-          {
-            "job_cluster_key": "job_cluster",
-            "notebook_task": {
-              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
-            },
-            "task_key": "notebook_task"
-          },
-          {
-            "depends_on": [
-              {
-                "task_key": "notebook_task"
-              }
-            ],
-            "pipeline_task": {
-              "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}"
-            },
-            "task_key": "refresh_pipeline"
-          },
           {
             "depends_on": [
               {
@@ -110,6 +92,24 @@
               "package_name": "project_name_[UNIQUE_NAME]"
             },
             "task_key": "main_task"
+          },
+          {
+            "job_cluster_key": "job_cluster",
+            "notebook_task": {
+              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
+            },
+            "task_key": "notebook_task"
+          },
+          {
+            "depends_on": [
+              {
+                "task_key": "notebook_task"
+              }
+            ],
+            "pipeline_task": {
+              "pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}"
+            },
+            "task_key": "refresh_pipeline"
           }
         ],
         "trigger": {
diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt
@@ -177,7 +177,7 @@ Validation OK!
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -79,5 +66,5 @@
+@@ -97,5 +84,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
@@ -335,7 +335,7 @@ Resources:
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -80,5 +67,5 @@
+@@ -98,5 +85,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
@@ -46,13 +46,6 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
         "dev": "[USERNAME]"
       },
       "tasks": [
-        {
-          "job_cluster_key": "job_cluster",
-          "notebook_task": {
-            "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/notebook"
-          },
-          "task_key": "notebook_task"
-        },
         {
           "depends_on": [
             {
@@ -70,6 +63,13 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
             "package_name": "my_jobs_as_code"
           },
           "task_key": "main_task"
+        },
+        {
+          "job_cluster_key": "job_cluster",
+          "notebook_task": {
+            "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/notebook"
+          },
+          "task_key": "notebook_task"
         }
       ],
       "trigger": {
diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks.go b/bundle/config/mutator/resourcemutator/merge_job_tasks.go
@@ -37,7 +37,11 @@ func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 		}
 
 		return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) {
-			return dyn.Map(job, "tasks", merge.ElementsByKey("task_key", m.taskKeyString))
+			// Sorting keys here since it'll be sorted by TF anyway
+			// https://github.com/databricks/terraform-provider-databricks/blob/0a932c2/jobs/resource_job.go#L343
+			// However, if we don't sort we have a difference between direct and TF and between configs in
+			// "bundle validate" and configs sent to backend.
+			return dyn.Map(job, "tasks", merge.ElementsBySortedKey("task_key", m.taskKeyString))
 		}))
 	})
 
diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks_test.go b/bundle/config/mutator/resourcemutator/merge_job_tasks_test.go
@@ -65,11 +65,11 @@ func TestMergeJobTasks(t *testing.T) {
 	j := b.Config.Resources.Jobs["foo"]
 
 	assert.Len(t, j.Tasks, 2)
-	assert.Equal(t, "foo", j.Tasks[0].TaskKey)
-	assert.Equal(t, "bar", j.Tasks[1].TaskKey)
+	assert.Equal(t, "foo", j.Tasks[1].TaskKey)
+	assert.Equal(t, "bar", j.Tasks[0].TaskKey)
 
 	// This task was merged with a subsequent one.
-	task0 := j.Tasks[0]
+	task0 := j.Tasks[1]
 	cluster := task0.NewCluster
 	assert.Equal(t, "13.3.x-scala2.12", cluster.SparkVersion)
 	assert.Equal(t, "i3.2xlarge", cluster.NodeTypeId)
@@ -79,7 +79,7 @@ func TestMergeJobTasks(t *testing.T) {
 	assert.Equal(t, "package2", task0.Libraries[1].Pypi.Package)
 
 	// This task was left untouched.
-	task1 := j.Tasks[1].NewCluster
+	task1 := j.Tasks[0].NewCluster
 	assert.Equal(t, "10.4.x-scala2.12", task1.SparkVersion)
 }
 
diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go
@@ -1,10 +1,15 @@
 package merge
 
-import "github.com/databricks/cli/libs/dyn"
+import (
+	"sort"
+
+	"github.com/databricks/cli/libs/dyn"
+)
 
 type elementsByKey struct {
-	key     string
-	keyFunc func(dyn.Value) string
+	key      string
+	keyFunc  func(dyn.Value) string
+	sortKeys bool
 }
 
 func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) {
@@ -42,6 +47,10 @@ func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Va
 		seen[key] = nv
 	}
 
+	if e.sortKeys {
+		sort.Strings(keys)
+	}
+
 	// Gather resulting elements in natural order.
 	out := make([]dyn.Value, 0, len(keys))
 	for _, key := range keys {
@@ -83,9 +92,13 @@ func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, erro
 // a parameter. The resulting elements get their key field overwritten
 // with the value as returned by the key function.
 func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
-	return elementsByKey{key, keyFunc}.Map
+	return elementsByKey{key, keyFunc, false}.Map
+}
+
+func ElementsBySortedKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
+	return elementsByKey{key, keyFunc, true}.Map
 }
 
 func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
-	return elementsByKey{key, keyFunc}.MapWithOverride
+	return elementsByKey{key, keyFunc, false}.MapWithOverride
 }

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,3 @@`
`1`		`-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # need to sort tasks by key like terraform does`
`2`		`-`
`3`	`1`	`[[Repls]]`
`4`	`2`	`Old = '\\\\'`
`5`	`3`	`New = '/'`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1 @@`
`1`	`1`	`BundleConfig.default_name = ""`
`2`		`-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # need to sort tasks by key`