Skip to content

Commit 8528ad3

Browse files
ilyakuz-dbilyakuz-db
authored
Add more server-side defaults to config sync (#4534)
## Changes 1. More resources default to the test server. 2. Filtering in config-remote-sync. Currently, defaults are hardcoded in config-remote-sync, will update in resource.yml once the config-remote-sync starts using bundle plan checker to determine skipped fields ## Why That was something that appeared in config-remote-sync when I tested it with different bundles ## Tests <!-- How have you tested the changes? --> Updated test-server, added more resources in acceptance tests <!-- If your PR needs to be included in the release notes for the next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent 49cb3a0 commit 8528ad3

File tree

18 files changed

+436
-49
lines changed

18 files changed

+436
-49
lines changed

‎acceptance/bundle/config-remote-sync/multiple_resources/databricks.yml.tmpl‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,32 @@ resources:
2424
spark_version: $DEFAULT_SPARK_VERSION
2525
node_type_id: $NODE_TYPE_ID
2626
num_workers: 1
27+
28+
clusters:
29+
test_cluster:
30+
cluster_name: cluster-$UNIQUE_NAME
31+
spark_version: $DEFAULT_SPARK_VERSION
32+
node_type_id: $NODE_TYPE_ID
33+
num_workers: 1
34+
35+
experiments:
36+
test_experiment:
37+
name: /Users/$CURRENT_USER_NAME/experiment-$UNIQUE_NAME
38+
39+
registered_models:
40+
test_model:
41+
name: model_$UNIQUE_NAME
42+
catalog_name: main
43+
schema_name: default
44+
45+
volumes:
46+
test_volume:
47+
name: volume_$UNIQUE_NAME
48+
catalog_name: main
49+
schema_name: default
50+
volume_type: MANAGED
51+
52+
sql_warehouses:
53+
test_warehouse:
54+
name: warehouse-$UNIQUE_NAME
55+
cluster_size: 2X-Small

‎acceptance/bundle/config-remote-sync/multiple_resources/output.txt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,29 @@ Resource: resources.jobs.job_two
3939
+ max_concurrent_runs: 10
4040
tasks:
4141
- task_key: main
42-
@@ -25,2 +27,4 @@
43-
node_type_id: [NODE_TYPE_ID]
42+
@@ -26,4 +28,6 @@
4443
num_workers: 1
44+
4545
+ tags:
4646
+ team: ml
47+
clusters:
48+
test_cluster:
4749

4850
>>> [CLI] bundle destroy --auto-approve
4951
The following resources will be deleted:
52+
delete resources.clusters.test_cluster
53+
delete resources.experiments.test_experiment
5054
delete resources.jobs.job_one
5155
delete resources.jobs.job_two
56+
delete resources.registered_models.test_model
57+
delete resources.sql_warehouses.test_warehouse
58+
delete resources.volumes.test_volume
59+
60+
This action will result in the deletion of the following volumes.
61+
For managed volumes, the files stored in the volume are also deleted from your
62+
cloud tenant within 30 days. For external volumes, the metadata about the volume
63+
is removed from the catalog, but the underlying files are not deleted:
64+
delete resources.volumes.test_volume
5265

5366
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
5467

‎acceptance/bundle/config-remote-sync/multiple_resources/script‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
envsubst < databricks.yml.tmpl > databricks.yml
44

55
cleanup() {
6+
# Restore original config before destroy to avoid Terraform errors
7+
# from server-side-only fields (e.g. creator_name) written by config-remote-sync.
8+
envsubst < databricks.yml.tmpl > databricks.yml
69
trace $CLI bundle destroy --auto-approve
710
}
811
trap cleanup EXIT
@@ -13,6 +16,9 @@ $CLI bundle deploy
1316
job_one_id="$(read_id.py job_one)"
1417
job_two_id="$(read_id.py job_two)"
1518

19+
# Add replacements for dynamic values that appear in server-side defaults
20+
add_repl.py "$($CLI current-user me | jq -r .id)" "USER_ID"
21+
add_repl.py "$($CLI metastores current | jq -r .metastore_id)" "METASTORE_ID"
1622

1723
title "Modify both jobs"
1824
edit_resource.py jobs $job_one_id <<EOF

‎acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,15 @@
3838
},
3939
"remote_state": {
4040
"autotermination_minutes": 60,
41+
"aws_attributes": {
42+
"availability": "SPOT_WITH_FALLBACK",
43+
"zone_id": "us-east-1c"
44+
},
4145
"cluster_id": "[CLUSTER_ID]",
4246
"cluster_name": "test-cluster-[UNIQUE_NAME]",
47+
"data_security_mode": "SINGLE_USER",
48+
"driver_node_type_id": "[NODE_TYPE_ID]",
49+
"enable_elastic_disk": false,
4350
"node_type_id": "[NODE_TYPE_ID]",
4451
"num_workers": 2,
4552
"spark_version": "13.3.x-snapshot-scala2.12"
@@ -52,6 +59,29 @@
5259
"min_workers": 2
5360
}
5461
},
62+
"aws_attributes": {
63+
"action": "skip",
64+
"reason": "server_side_default",
65+
"remote": {
66+
"availability": "SPOT_WITH_FALLBACK",
67+
"zone_id": "us-east-1c"
68+
}
69+
},
70+
"data_security_mode": {
71+
"action": "skip",
72+
"reason": "server_side_default",
73+
"remote": "SINGLE_USER"
74+
},
75+
"driver_node_type_id": {
76+
"action": "skip",
77+
"reason": "server_side_default",
78+
"remote": "[NODE_TYPE_ID]"
79+
},
80+
"enable_elastic_disk": {
81+
"action": "skip",
82+
"reason": "empty",
83+
"remote": false
84+
},
5585
"num_workers": {
5686
"action": "update",
5787
"old": 2,
@@ -87,8 +117,15 @@
87117
"min_workers": 2
88118
},
89119
"autotermination_minutes": 60,
120+
"aws_attributes": {
121+
"availability": "SPOT_WITH_FALLBACK",
122+
"zone_id": "us-east-1c"
123+
},
90124
"cluster_id": "[CLUSTER_ID]",
91125
"cluster_name": "test-cluster-[UNIQUE_NAME]",
126+
"data_security_mode": "SINGLE_USER",
127+
"driver_node_type_id": "[NODE_TYPE_ID]",
128+
"enable_elastic_disk": false,
92129
"node_type_id": "[NODE_TYPE_ID]",
93130
"spark_version": "13.3.x-snapshot-scala2.12"
94131
},
@@ -104,6 +141,29 @@
104141
"old": 2,
105142
"new": 3,
106143
"remote": 2
144+
},
145+
"aws_attributes": {
146+
"action": "skip",
147+
"reason": "server_side_default",
148+
"remote": {
149+
"availability": "SPOT_WITH_FALLBACK",
150+
"zone_id": "us-east-1c"
151+
}
152+
},
153+
"data_security_mode": {
154+
"action": "skip",
155+
"reason": "server_side_default",
156+
"remote": "SINGLE_USER"
157+
},
158+
"driver_node_type_id": {
159+
"action": "skip",
160+
"reason": "server_side_default",
161+
"remote": "[NODE_TYPE_ID]"
162+
},
163+
"enable_elastic_disk": {
164+
"action": "skip",
165+
"reason": "empty",
166+
"remote": false
107167
}
108168
}
109169
}
@@ -135,8 +195,15 @@
135195
"min_workers": 3
136196
},
137197
"autotermination_minutes": 60,
198+
"aws_attributes": {
199+
"availability": "SPOT_WITH_FALLBACK",
200+
"zone_id": "us-east-1c"
201+
},
138202
"cluster_id": "[CLUSTER_ID]",
139203
"cluster_name": "test-cluster-[UNIQUE_NAME]",
204+
"data_security_mode": "SINGLE_USER",
205+
"driver_node_type_id": "[NODE_TYPE_ID]",
206+
"enable_elastic_disk": false,
140207
"node_type_id": "[NODE_TYPE_ID]",
141208
"spark_version": "13.3.x-snapshot-scala2.12",
142209
"state": "RUNNING"
@@ -155,6 +222,29 @@
155222
"old": 3,
156223
"new": 4,
157224
"remote": 3
225+
},
226+
"aws_attributes": {
227+
"action": "skip",
228+
"reason": "server_side_default",
229+
"remote": {
230+
"availability": "SPOT_WITH_FALLBACK",
231+
"zone_id": "us-east-1c"
232+
}
233+
},
234+
"data_security_mode": {
235+
"action": "skip",
236+
"reason": "server_side_default",
237+
"remote": "SINGLE_USER"
238+
},
239+
"driver_node_type_id": {
240+
"action": "skip",
241+
"reason": "server_side_default",
242+
"remote": "[NODE_TYPE_ID]"
243+
},
244+
"enable_elastic_disk": {
245+
"action": "skip",
246+
"reason": "empty",
247+
"remote": false
158248
}
159249
}
160250
}
@@ -183,8 +273,15 @@
183273
"min_workers": 4
184274
},
185275
"autotermination_minutes": 60,
276+
"aws_attributes": {
277+
"availability": "SPOT_WITH_FALLBACK",
278+
"zone_id": "us-east-1c"
279+
},
186280
"cluster_id": "[CLUSTER_ID]",
187281
"cluster_name": "test-cluster-[UNIQUE_NAME]",
282+
"data_security_mode": "SINGLE_USER",
283+
"driver_node_type_id": "[NODE_TYPE_ID]",
284+
"enable_elastic_disk": false,
188285
"node_type_id": "[NODE_TYPE_ID]",
189286
"spark_version": "13.3.x-snapshot-scala2.12",
190287
"state": "RUNNING"
@@ -202,6 +299,29 @@
202299
"min_workers": 4
203300
}
204301
},
302+
"aws_attributes": {
303+
"action": "skip",
304+
"reason": "server_side_default",
305+
"remote": {
306+
"availability": "SPOT_WITH_FALLBACK",
307+
"zone_id": "us-east-1c"
308+
}
309+
},
310+
"data_security_mode": {
311+
"action": "skip",
312+
"reason": "server_side_default",
313+
"remote": "SINGLE_USER"
314+
},
315+
"driver_node_type_id": {
316+
"action": "skip",
317+
"reason": "server_side_default",
318+
"remote": "[NODE_TYPE_ID]"
319+
},
320+
"enable_elastic_disk": {
321+
"action": "skip",
322+
"reason": "empty",
323+
"remote": false
324+
},
205325
"num_workers": {
206326
"action": "resize",
207327
"reason": "custom",

‎acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ Deploying resources...
2020
Updating deployment state...
2121
Deployment complete!
2222

23-
>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) out.requests.txt
23+
>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt
2424
{
2525
"method": "POST",
2626
"path": "/api/2.1/clusters/edit",
@@ -55,7 +55,7 @@ Deploying resources...
5555
Updating deployment state...
5656
Deployment complete!
5757

58-
>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) out.requests.txt
58+
>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt
5959
{
6060
"method": "POST",
6161
"path": "/api/2.1/clusters/edit",
@@ -89,8 +89,15 @@ Deployment complete!
8989
"min_workers":3
9090
},
9191
"autotermination_minutes":60,
92+
"aws_attributes": {
93+
"availability":"SPOT_WITH_FALLBACK",
94+
"zone_id":"us-east-1c"
95+
},
9296
"cluster_id":"[CLUSTER_ID]",
9397
"cluster_name":"test-cluster-[UNIQUE_NAME]",
98+
"data_security_mode":"SINGLE_USER",
99+
"driver_node_type_id":"[NODE_TYPE_ID]",
100+
"enable_elastic_disk":false,
94101
"node_type_id":"[NODE_TYPE_ID]",
95102
"spark_version":"13.3.x-snapshot-scala2.12",
96103
"state":"RUNNING"

‎acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ update_file.py databricks.yml " num_workers: 2" " autoscale:
2222
$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt
2323
$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json
2424
trace $CLI bundle deploy
25-
trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt
25+
trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt
2626
rm out.requests.txt
2727

2828
title "Cluster should have autoscale\n"
@@ -34,7 +34,7 @@ update_file.py databricks.yml "max_workers: 4" "max_workers: 5"
3434
$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt
3535
$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json
3636
trace $CLI bundle deploy
37-
trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt
37+
trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt
3838
rm out.requests.txt
3939

4040
title "Cluster should have new autoscale\n"

0 commit comments

Comments
 (0)