Fix lookup precedence bug (#4916)

## Changes
Remove lookup if explicit override exists for target

## Why
Fixes #4915 


[Documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/variables#precedence-order)
says the order is:
4. Within any `variables` mappings, among the `targets` mappings within
your bundle configuration files.
5. Any `default` value for that variable's definition, among the
top-level `variables` mappings within your bundle configuration files.

## Tests
Added unit and acceptance tests

---------

Co-authored-by: simon <4305831+simonfaltum@users.noreply.github.com>

Author: janniklasrose

acceptance/bundle/variables/variable_overrides_in_target/databricks.yml

@@ -22,6 +22,11 @@ variables:
     default: true
     description: "A boolean variable"
 
+  w_lookup:
+    description: "A variable with a lookup"
+    lookup:
+      cluster: "some-test-cluster" # id 4321
+
 targets:
   use-default-variable-values:
 
@@ -37,3 +42,13 @@ targets:
     variables:
       foo: "overridden_string"
       baz: false
+
+  override-lookup-variable-with-value:
+    variables:
+      w_lookup: "my-specific-cluster-id"
+
+  override-lookup-variable-with-lookup:
+    variables:
+      w_lookup:
+        lookup:
+          cluster: "some-other-cluster" # id 9876

acceptance/bundle/variables/variable_overrides_in_target/output.txt

@@ -86,3 +86,9 @@
     }
   }
 }
+
+>>> [CLI] bundle validate -o json -t override-lookup-variable-with-value
+"my-specific-cluster-id"
+
+>>> [CLI] bundle validate -o json -t override-lookup-variable-with-lookup
+"9876"

acceptance/bundle/variables/variable_overrides_in_target/script

@@ -2,3 +2,5 @@ trace $CLI bundle validate -o json -t use-default-variable-values | jq .resource
 trace $CLI bundle validate -o json -t override-string-variable | jq .resources
 trace $CLI bundle validate -o json -t override-int-variable | jq .resources
 trace $CLI bundle validate -o json -t override-both-bool-and-string-variables | jq .resources
+trace $CLI bundle validate -o json -t override-lookup-variable-with-value | jq .variables.w_lookup.value
+trace $CLI bundle validate -o json -t override-lookup-variable-with-lookup | jq .variables.w_lookup.value

bundle/config/root.go

@@ -347,12 +347,28 @@ func (r *Root) MergeTargetOverrides(name string) error {
 			if vDefault.Kind() != dyn.KindInvalid {
 				defaultPath := varPath.Append(dyn.Key("default"))
 				root, err = dyn.SetByPath(root, defaultPath, vDefault)
+				if err != nil {
+					return root, err
+				}
+
+				// If the target explicitly sets a default value, drop any lookup from the
+				// root variable definition so SetVariables can assign this default.
+				lookupPath := varPath.Append(dyn.Key("lookup"))
+				root, err = dyn.SetByPath(root, lookupPath, dyn.NilValue)
 			}
 
 			vLookup := variable.Get("lookup")
 			if vLookup.Kind() != dyn.KindInvalid {
 				lookupPath := varPath.Append(dyn.Key("lookup"))
 				root, err = dyn.SetByPath(root, lookupPath, vLookup)
+				if err != nil {
+					return root, err
+				}
+
+				// If the target explicitly sets a lookup, drop any default value from the
+				// root variable definition so lookup resolution remains authoritative.
+				defaultPath := varPath.Append(dyn.Key("default"))
+				root, err = dyn.SetByPath(root, defaultPath, dyn.NilValue)
 			}
 
 			return root, err