Skip to content

Commit b60b9f0

Browse files
shreyas-goenkashreyas-goenka
committed
update tests
2 parents 8c04119 + a93fc07 commit b60b9f0

File tree

84 files changed

+3579
-4614
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+3579
-4614
lines changed

‎acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json‎

Lines changed: 63 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -77,77 +77,75 @@
7777
"remote_state": {
7878
"created_time": [UNIX_TIME_MILLIS],
7979
"creator_user_name": "[USERNAME]",
80+
"deployment": {
81+
"kind": "BUNDLE",
82+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
83+
},
84+
"edit_mode": "UI_LOCKED",
85+
"email_notifications": {},
86+
"environments": [
87+
{
88+
"environment_key": "test_env",
89+
"spec": {
90+
"client": "1",
91+
"dependencies": [
92+
"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
93+
]
94+
}
95+
}
96+
],
97+
"format": "MULTI_TASK",
8098
"job_id": [NUMID],
99+
"max_concurrent_runs": 1,
100+
"name": "[default] My Wheel Job",
101+
"queue": {
102+
"enabled": true
103+
},
81104
"run_as_user_name": "[USERNAME]",
82-
"settings": {
83-
"deployment": {
84-
"kind": "BUNDLE",
85-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
105+
"tasks": [
106+
{
107+
"environment_key": "test_env",
108+
"python_wheel_task": {
109+
"entry_point": "run",
110+
"package_name": "my_test_code"
111+
},
112+
"task_key": "ServerlessTestTask"
86113
},
87-
"edit_mode": "UI_LOCKED",
88-
"email_notifications": {},
89-
"environments": [
90-
{
91-
"environment_key": "test_env",
92-
"spec": {
93-
"client": "1",
94-
"dependencies": [
95-
"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
96-
]
114+
{
115+
"existing_cluster_id": "0717-132531-5opeqon1",
116+
"for_each_task": {
117+
"inputs": "[1]",
118+
"task": {
119+
"existing_cluster_id": "0717-132531-5opeqon1",
120+
"libraries": [
121+
{
122+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
123+
}
124+
],
125+
"python_wheel_task": {
126+
"entry_point": "run",
127+
"package_name": "my_test_code"
128+
},
129+
"task_key": "SubTask"
97130
}
98-
}
99-
],
100-
"format": "MULTI_TASK",
101-
"max_concurrent_runs": 1,
102-
"name": "[default] My Wheel Job",
103-
"queue": {
104-
"enabled": true
105-
},
106-
"tasks": [
107-
{
108-
"environment_key": "test_env",
109-
"python_wheel_task": {
110-
"entry_point": "run",
111-
"package_name": "my_test_code"
112-
},
113-
"task_key": "ServerlessTestTask"
114131
},
115-
{
116-
"existing_cluster_id": "0717-132531-5opeqon1",
117-
"for_each_task": {
118-
"inputs": "[1]",
119-
"task": {
120-
"existing_cluster_id": "0717-132531-5opeqon1",
121-
"libraries": [
122-
{
123-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
124-
}
125-
],
126-
"python_wheel_task": {
127-
"entry_point": "run",
128-
"package_name": "my_test_code"
129-
},
130-
"task_key": "SubTask"
131-
}
132-
},
133-
"libraries": [
134-
{
135-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
136-
},
137-
{
138-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
139-
}
140-
],
141-
"python_wheel_task": {
142-
"entry_point": "run",
143-
"package_name": "my_test_code"
132+
"libraries": [
133+
{
134+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
144135
},
145-
"task_key": "TestTask"
146-
}
147-
],
148-
"timeout_seconds": 0,
149-
"webhook_notifications": {}
150-
}
136+
{
137+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
138+
}
139+
],
140+
"python_wheel_task": {
141+
"entry_point": "run",
142+
"package_name": "my_test_code"
143+
},
144+
"task_key": "TestTask"
145+
}
146+
],
147+
"timeout_seconds": 0,
148+
"webhook_notifications": {}
151149
},
152150
"changes": {
153151
"email_notifications": {

‎acceptance/bundle/bundle_tag/url_ref/out.deploy.direct.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
2-
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in jobs.Job
2+
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in dresources.JobRemote
33

44
Error: planning failed
55

‎acceptance/bundle/bundle_tag/url_ref/out.plan.direct.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in jobs.Job
1+
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in dresources.JobRemote
22

33
Error: planning failed
44

‎acceptance/bundle/config-remote-sync/job_multiple_tasks/databricks.yml.tmpl‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,56 +5,72 @@ resources:
55
jobs:
66
my_job:
77
tasks:
8-
- task_key: task1
8+
- task_key: b_task
99
notebook_task:
10-
notebook_path: /Users/{{workspace_user_name}}/task1
10+
notebook_path: /Users/{{workspace_user_name}}/b_task
1111
new_cluster:
1212
spark_version: $DEFAULT_SPARK_VERSION
1313
node_type_id: $NODE_TYPE_ID
1414
num_workers: 1
15-
- task_key: task2
15+
- task_key: d_task
1616
notebook_task:
17-
notebook_path: /Users/{{workspace_user_name}}/task2
17+
notebook_path: /Users/{{workspace_user_name}}/d_task
1818
new_cluster:
1919
spark_version: $DEFAULT_SPARK_VERSION
2020
node_type_id: $NODE_TYPE_ID
2121
num_workers: 2
2222
depends_on:
23-
- task_key: task1
24-
- task_key: task3
23+
- task_key: b_task
24+
- task_key: c_task
2525
notebook_task:
26-
notebook_path: /Users/{{workspace_user_name}}/task3
26+
notebook_path: /Users/{{workspace_user_name}}/c_task
2727
new_cluster:
2828
spark_version: $DEFAULT_SPARK_VERSION
2929
node_type_id: $NODE_TYPE_ID
3030
num_workers: 2
3131
depends_on:
32-
- task_key: task2
33-
- task_key: task4
32+
- task_key: d_task
33+
- task_key: a_task
3434
notebook_task:
35-
notebook_path: /Users/{{workspace_user_name}}/task4
35+
notebook_path: /Users/{{workspace_user_name}}/a_task
3636
new_cluster:
3737
spark_version: $DEFAULT_SPARK_VERSION
3838
node_type_id: $NODE_TYPE_ID
3939
num_workers: 1
4040
depends_on:
41-
- task_key: task3
41+
- task_key: c_task
4242

4343
rename_task_job:
4444
tasks:
45-
- task_key: task_rename_1
45+
- task_key: b_task
4646
notebook_task:
47-
notebook_path: /Users/{{workspace_user_name}}/rename_task_1
47+
notebook_path: /Users/{{workspace_user_name}}/b_task
4848
new_cluster:
4949
spark_version: $DEFAULT_SPARK_VERSION
5050
node_type_id: $NODE_TYPE_ID
5151
num_workers: 1
52-
- task_key: task_rename_2
52+
- task_key: d_task
53+
depends_on:
54+
- task_key: b_task
5355
notebook_task:
54-
notebook_path: /Users/{{workspace_user_name}}/rename_task_2
56+
notebook_path: /Users/{{workspace_user_name}}/d_task
5557
new_cluster:
5658
spark_version: $DEFAULT_SPARK_VERSION
5759
node_type_id: $NODE_TYPE_ID
5860
num_workers: 1
61+
- task_key: c_task
5962
depends_on:
60-
- task_key: task_rename_1
63+
- task_key: b_task
64+
notebook_task:
65+
notebook_path: /Users/{{workspace_user_name}}/c_task
66+
new_cluster:
67+
spark_version: $DEFAULT_SPARK_VERSION
68+
node_type_id: $NODE_TYPE_ID
69+
num_workers: 1
70+
- task_key: a_task
71+
notebook_task:
72+
notebook_path: /Users/{{workspace_user_name}}/a_task
73+
new_cluster:
74+
spark_version: $DEFAULT_SPARK_VERSION
75+
node_type_id: $NODE_TYPE_ID
76+
num_workers: 1

‎acceptance/bundle/config-remote-sync/job_multiple_tasks/output.txt‎

Lines changed: 37 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ Deploying resources...
33
Updating deployment state...
44
Deployment complete!
55

6-
=== Modify only 'process' task num_workers and add timeout
6+
=== Modify c_task, remove d_task, add e_task
77
=== Detect and save changes
88
Detected changes in 1 resource(s):
99

1010
Resource: resources.jobs.my_job
11-
tasks[task_key='new_task']: add
12-
tasks[task_key='task2']: remove
13-
tasks[task_key='task3'].depends_on[0].task_key: replace
14-
tasks[task_key='task3'].new_cluster.num_workers: replace
15-
tasks[task_key='task3'].timeout_seconds: add
11+
tasks[task_key='c_task'].depends_on[0].task_key: replace
12+
tasks[task_key='c_task'].new_cluster.num_workers: replace
13+
tasks[task_key='c_task'].timeout_seconds: add
14+
tasks[task_key='d_task']: remove
15+
tasks[task_key='e_task']: add
1616

1717

1818

19-
=== Configuration changes for new task
19+
=== Configuration changes
2020

2121
>>> diff.py databricks.yml.backup databricks.yml
2222
--- databricks.yml.backup
@@ -30,37 +30,37 @@ Resource: resources.jobs.my_job
3030
@@ -13,13 +12,11 @@
3131
node_type_id: [NODE_TYPE_ID]
3232
num_workers: 1
33-
- - task_key: task2
33+
- - task_key: d_task
3434
+ - new_cluster:
3535
+ node_type_id: [NODE_TYPE_ID]
3636
+ num_workers: 1
3737
+ spark_version: 13.3.x-snapshot-scala2.12
3838
notebook_task:
39-
- notebook_path: /Users/{{workspace_user_name}}/task2
39+
- notebook_path: /Users/{{workspace_user_name}}/d_task
4040
- new_cluster:
4141
- spark_version: 13.3.x-snapshot-scala2.12
4242
- node_type_id: [NODE_TYPE_ID]
4343
- num_workers: 2
4444
- depends_on:
45-
- - task_key: task1
46-
+ notebook_path: /Users/[USERNAME]/new_task
47-
+ task_key: new_task
48-
- task_key: task3
45+
- - task_key: b_task
46+
+ notebook_path: /Users/[USERNAME]/e_task
47+
+ task_key: e_task
48+
- task_key: c_task
4949
notebook_task:
5050
@@ -28,7 +25,8 @@
5151
spark_version: 13.3.x-snapshot-scala2.12
5252
node_type_id: [NODE_TYPE_ID]
5353
- num_workers: 2
5454
+ num_workers: 5
5555
depends_on:
56-
- - task_key: task2
57-
+ - task_key: task1
56+
- - task_key: d_task
57+
+ - task_key: b_task
5858
+ timeout_seconds: 3600
59-
- task_key: task4
59+
- task_key: a_task
6060
notebook_task:
6161
@@ -40,5 +38,4 @@
6262
depends_on:
63-
- task_key: task3
63+
- task_key: c_task
6464
-
6565
rename_task_job:
6666
tasks:
@@ -69,14 +69,15 @@ Deploying resources...
6969
Updating deployment state...
7070
Deployment complete!
7171

72-
=== Rename task_rename_1 to task_rename_new
72+
=== Rename b_task to b_task_renamed (4 tasks, 2 with depends_on, 1 without)
7373
=== Detect task key rename
7474
Detected changes in 1 resource(s):
7575

7676
Resource: resources.jobs.rename_task_job
77-
tasks[task_key='task_rename_1']: remove
78-
tasks[task_key='task_rename_2'].depends_on[0].task_key: replace
79-
tasks[task_key='task_rename_new']: add
77+
tasks[task_key='b_task']: remove
78+
tasks[task_key='b_task_renamed']: add
79+
tasks[task_key='c_task'].depends_on[0].task_key: replace
80+
tasks[task_key='d_task'].depends_on[0].task_key: replace
8081

8182

8283

@@ -85,25 +86,31 @@ Resource: resources.jobs.rename_task_job
8586
>>> diff.py databricks.yml.backup2 databricks.yml
8687
--- databricks.yml.backup2
8788
+++ databricks.yml
88-
@@ -40,11 +40,11 @@
89+
@@ -40,14 +40,14 @@
8990
rename_task_job:
9091
tasks:
91-
- - task_key: task_rename_1
92+
- - task_key: b_task
9293
+ - new_cluster:
9394
+ node_type_id: [NODE_TYPE_ID]
9495
+ num_workers: 1
9596
+ spark_version: 13.3.x-snapshot-scala2.12
9697
notebook_task:
97-
notebook_path: /Users/{{workspace_user_name}}/rename_task_1
98+
notebook_path: /Users/{{workspace_user_name}}/b_task
9899
- new_cluster:
99100
- spark_version: 13.3.x-snapshot-scala2.12
100101
- node_type_id: [NODE_TYPE_ID]
101102
- num_workers: 1
102-
+ task_key: task_rename_new
103-
- task_key: task_rename_2
103+
+ task_key: b_task_renamed
104+
- task_key: d_task
105+
depends_on:
106+
- - task_key: b_task
107+
+ - task_key: b_task_renamed
104108
notebook_task:
105-
@@ -55,3 +55,3 @@
106-
num_workers: 1
109+
notebook_path: /Users/{{workspace_user_name}}/d_task
110+
@@ -58,5 +58,5 @@
111+
- task_key: c_task
107112
depends_on:
108-
- - task_key: task_rename_1
109-
+ - task_key: task_rename_new
113+
- - task_key: b_task
114+
+ - task_key: b_task_renamed
115+
notebook_task:
116+
notebook_path: /Users/{{workspace_user_name}}/c_task

0 commit comments

Comments
 (0)