Skip to content

Commit 51e5c28

Browse files
denikdenik
authored
Add volumes support to direct & testserver (#3211)
## Changes Add volumes support to direct deployment and testserver ## Tests New acceptance test.
1 parent a1b5daf commit 51e5c28

File tree

11 files changed

+303
-2
lines changed

11 files changed

+303
-2
lines changed

‎acceptance/bundle/deploy/volume/recreate/test.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ Local = false
22
Cloud = true
33
RequiresUnityCatalog = true
44

5-
EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # volumes are not supported
5+
EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # needs volume recreation, grants
66

77
Ignore = [
88
"databricks.yml",

‎acceptance/bundle/resources/volumes/databricks.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
resources:
2+
volumes:
3+
volume1:
4+
catalog_name: main
5+
schema_name: myschema
6+
name: myvolume
7+
comment: COMMENT1

‎acceptance/bundle/resources/volumes/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Local = true
2+
Cloud = false
3+
4+
[EnvMatrix]
5+
DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

‎acceptance/bundle/resources/volumes/output.txt‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
2+
=== Initial summary for resource, before the deployment
3+
>>> [CLI] bundle summary -o json
4+
{
5+
"catalog_name": "main",
6+
"comment": "COMMENT1",
7+
"modified_status": "created",
8+
"name": "myvolume",
9+
"schema_name": "myschema",
10+
"volume_type": "MANAGED"
11+
}
12+
13+
=== Verify volume does not exist
14+
>>> musterr [CLI] volumes read main.myschema.myvolume
15+
Error: Resource catalog.VolumeInfo not found: main.myschema.myvolume
16+
17+
Exit code (musterr): 1
18+
19+
>>> [CLI] bundle deploy
20+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
21+
Deploying resources...
22+
Updating deployment state...
23+
Deployment complete!
24+
25+
>>> print_requests
26+
{
27+
"method": "POST",
28+
"path": "/api/2.1/unity-catalog/volumes",
29+
"body": {
30+
"catalog_name": "main",
31+
"comment": "COMMENT1",
32+
"name": "myvolume",
33+
"schema_name": "myschema",
34+
"volume_type": "MANAGED"
35+
}
36+
}
37+
38+
=== Summary should now show id and url
39+
"main.myschema.myvolume"
40+
"[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?o=[NUMID]"
41+
42+
=== Verify deployment
43+
>>> [CLI] volumes read main.myschema.myvolume
44+
{
45+
"catalog_name":"main",
46+
"comment":"COMMENT1",
47+
"full_name":"main.myschema.myvolume",
48+
"name":"myvolume",
49+
"schema_name":"myschema",
50+
"volume_type":"MANAGED"
51+
}
52+
53+
=== Update comment
54+
>>> update_file.py databricks.yml COMMENT1 COMMENT2
55+
56+
=== Summary should show modified_status=modified and show the same id
57+
{
58+
"catalog_name": "main",
59+
"comment": "COMMENT2",
60+
"id": "main.myschema.myvolume",
61+
"name": "myvolume",
62+
"schema_name": "myschema",
63+
"url": "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?o=[NUMID]",
64+
"volume_type": "MANAGED"
65+
}
66+
67+
>>> [CLI] bundle deploy
68+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
69+
Deploying resources...
70+
Updating deployment state...
71+
Deployment complete!
72+
73+
>>> print_requests
74+
{
75+
"method": "PATCH",
76+
"path": "/api/2.1/unity-catalog/volumes/main.myschema.myvolume",
77+
"body": {
78+
"comment": "COMMENT2"
79+
}
80+
}
81+
82+
=== Verify updated deployment: should show new comment
83+
>>> [CLI] volumes read main.myschema.myvolume
84+
"COMMENT2"
85+
86+
>>> [CLI] bundle destroy --auto-approve
87+
The following resources will be deleted:
88+
delete volume volume1
89+
90+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
91+
92+
Deleting files...
93+
Destroy complete!
94+
95+
>>> print_requests
96+
{
97+
"method": "DELETE",
98+
"path": "/api/2.1/unity-catalog/volumes/main.myschema.myvolume"
99+
}
100+
101+
>>> musterr [CLI] volumes read main.myschema.myvolume
102+
Error: Resource catalog.VolumeInfo not found: main.myschema.myvolume
103+
104+
Exit code (musterr): 1
105+
{
106+
"catalog_name": "main",
107+
"comment": "COMMENT2",
108+
"modified_status": "created",
109+
"name": "myvolume",
110+
"schema_name": "myschema",
111+
"volume_type": "MANAGED"
112+
}

‎acceptance/bundle/resources/volumes/script‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
title "Initial summary for resource, before the deployment"
2+
trace $CLI bundle summary -o json | jq .resources.volumes.volume1
3+
4+
title "Verify volume does not exist"
5+
trace musterr $CLI volumes read main.myschema.myvolume
6+
7+
trace $CLI bundle deploy
8+
9+
print_requests() {
10+
jq 'select(.method != "GET" and (.path | contains("/unity")))' < out.requests.txt
11+
rm out.requests.txt
12+
}
13+
14+
trace print_requests
15+
16+
title "Summary should now show id and url\n"
17+
$CLI bundle summary -o json | jq .resources.volumes.volume1.id
18+
$CLI bundle summary -o json | jq .resources.volumes.volume1.url
19+
20+
title "Verify deployment"
21+
trace $CLI volumes read main.myschema.myvolume
22+
23+
title "Update comment"
24+
trace update_file.py databricks.yml COMMENT1 COMMENT2
25+
26+
title "Summary should show modified_status=modified and show the same id\n"
27+
$CLI bundle summary -o json | jq .resources.volumes.volume1
28+
29+
trace $CLI bundle deploy
30+
trace print_requests
31+
32+
title "Verify updated deployment: should show new comment"
33+
trace $CLI volumes read main.myschema.myvolume | jq .comment
34+
35+
trace $CLI bundle destroy --auto-approve
36+
trace print_requests
37+
38+
trace musterr $CLI volumes read main.myschema.myvolume
39+
$CLI bundle summary -o json | jq .resources.volumes.volume1
40+
41+
rm out.requests.txt

‎acceptance/bundle/resources/volumes/test.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Ignore = [
2+
"tmp.summary.txt",
3+
".databricks",
4+
]

‎acceptance/internal/handlers.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,21 @@ func addDefaultHandlers(server *testserver.Server) {
378378
return testserver.MapDelete(req.Workspace, req.Workspace.Schemas, req.Vars["full_name"])
379379
})
380380

381+
// Volumes:
382+
383+
server.Handle("GET", "/api/2.1/unity-catalog/volumes/{full_name}", func(req testserver.Request) any {
384+
return testserver.MapGet(req.Workspace, req.Workspace.Volumes, req.Vars["full_name"])
385+
})
386+
381387
server.Handle("POST", "/api/2.1/unity-catalog/volumes", func(req testserver.Request) any {
382388
return req.Workspace.VolumesCreate(req)
383389
})
390+
391+
server.Handle("PATCH", "/api/2.1/unity-catalog/volumes/{full_name}", func(req testserver.Request) any {
392+
return req.Workspace.VolumesUpdate(req, req.Vars["full_name"])
393+
})
394+
395+
server.Handle("DELETE", "/api/2.1/unity-catalog/volumes/{full_name}", func(req testserver.Request) any {
396+
return testserver.MapDelete(req.Workspace, req.Workspace.Volumes, req.Vars["full_name"])
397+
})
384398
}

‎bundle/terranova/tnresources/resource.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,15 @@ const (
1515
_jobs = "jobs"
1616
_pipelines = "pipelines"
1717
_schemas = "schemas"
18+
_volumes = "volumes"
1819
_apps = "apps"
1920
)
2021

2122
var supportedResources = map[string]reflect.Value{
2223
_jobs: reflect.ValueOf(NewResourceJob),
2324
_pipelines: reflect.ValueOf(NewResourcePipeline),
2425
_schemas: reflect.ValueOf(NewResourceSchema),
26+
_volumes: reflect.ValueOf(NewResourceVolume),
2527
_apps: reflect.ValueOf(NewResourceApp),
2628
}
2729

@@ -30,6 +32,7 @@ var supportedResourcesTypes = map[string]reflect.Type{
3032
_jobs: reflect.TypeOf(ResourceJob{}.config),
3133
_pipelines: reflect.TypeOf(ResourcePipeline{}.config),
3234
_schemas: reflect.TypeOf(ResourceSchema{}.config),
35+
_volumes: reflect.TypeOf(ResourceVolume{}.config),
3336
_apps: reflect.TypeOf(ResourceApp{}.config),
3437
}
3538

@@ -39,6 +42,7 @@ var deletableResources = map[string]DeleteResourceFN{
3942
_jobs: DeleteJob,
4043
_pipelines: DeletePipeline,
4144
_schemas: DeleteSchema,
45+
_volumes: DeleteVolume,
4246
_apps: DeleteApp,
4347
}
4448

‎bundle/terranova/tnresources/volume.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package tnresources
2+
3+
import (
4+
"context"
5+
6+
"github.com/databricks/cli/bundle/config/resources"
7+
"github.com/databricks/cli/bundle/deployplan"
8+
"github.com/databricks/cli/libs/structdiff"
9+
"github.com/databricks/databricks-sdk-go"
10+
"github.com/databricks/databricks-sdk-go/service/catalog"
11+
)
12+
13+
type ResourceVolume struct {
14+
client *databricks.WorkspaceClient
15+
config catalog.CreateVolumeRequestContent
16+
}
17+
18+
func NewResourceVolume(client *databricks.WorkspaceClient, schema *resources.Volume) (*ResourceVolume, error) {
19+
return &ResourceVolume{
20+
client: client,
21+
config: schema.CreateVolumeRequestContent,
22+
}, nil
23+
}
24+
25+
func (r *ResourceVolume) Config() any {
26+
return r.config
27+
}
28+
29+
func (r *ResourceVolume) DoCreate(ctx context.Context) (string, error) {
30+
response, err := r.client.Volumes.Create(ctx, r.config)
31+
if err != nil {
32+
return "", SDKError{Method: "Volumes.Create", Err: err}
33+
}
34+
return response.FullName, nil
35+
}
36+
37+
func (r *ResourceVolume) DoUpdate(ctx context.Context, id string) (string, error) {
38+
updateRequest := catalog.UpdateVolumeRequestContent{}
39+
err := copyViaJSON(&updateRequest, r.config)
40+
if err != nil {
41+
return "", err
42+
}
43+
44+
updateRequest.Name = id
45+
46+
response, err := r.client.Volumes.Update(ctx, updateRequest)
47+
if err != nil {
48+
return "", SDKError{Method: "Volumes.Update", Err: err}
49+
}
50+
51+
return response.FullName, nil
52+
}
53+
54+
func DeleteVolume(ctx context.Context, client *databricks.WorkspaceClient, id string) error {
55+
err := client.Volumes.DeleteByName(ctx, id)
56+
if err != nil {
57+
return SDKError{Method: "Volumes.Delete", Err: err}
58+
}
59+
return nil
60+
}
61+
62+
func (r *ResourceVolume) WaitAfterCreate(ctx context.Context) error {
63+
// Intentional no-op
64+
return nil
65+
}
66+
67+
func (r *ResourceVolume) WaitAfterUpdate(ctx context.Context) error {
68+
// Intentional no-op
69+
return nil
70+
}
71+
72+
func (r *ResourceVolume) ClassifyChanges(changes []structdiff.Change) deployplan.ActionType {
73+
// TODO: Name, SchemaName changes should result in re-create
74+
return deployplan.ActionTypeUpdate
75+
}

‎libs/testserver/fake_workspace.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ type FakeWorkspace struct {
5151
Monitors map[string]catalog.MonitorInfo
5252
Apps map[string]apps.App
5353
Schemas map[string]catalog.SchemaInfo
54+
Volumes map[string]catalog.VolumeInfo
5455
Dashboards map[string]dashboards.Dashboard
5556
}
5657

@@ -125,6 +126,7 @@ func NewFakeWorkspace(url string) *FakeWorkspace {
125126
Monitors: map[string]catalog.MonitorInfo{},
126127
Apps: map[string]apps.App{},
127128
Schemas: map[string]catalog.SchemaInfo{},
129+
Volumes: map[string]catalog.VolumeInfo{},
128130
Dashboards: map[string]dashboards.Dashboard{},
129131
}
130132
}

0 commit comments

Comments
 (0)