diff --git a/acceptance/apps/deploy/bundle-no-args/output.txt b/acceptance/apps/deploy/bundle-no-args/output.txt index 18509ae92f..d6c98f8535 100644 --- a/acceptance/apps/deploy/bundle-no-args/output.txt +++ b/acceptance/apps/deploy/bundle-no-args/output.txt @@ -7,7 +7,10 @@ Updating deployment state... Deployment complete! ✓ Getting the status of the app myapp ✓ App is in RUNNING state -✓ App compute is in ACTIVE state +✓ App compute is in STOPPED state +✓ Starting the app myapp +✓ App is starting... +✓ App is started! ✓ Deployment succeeded You can access the app at myapp-123.cloud.databricksapps.com ✔ Deployment complete! diff --git a/acceptance/bundle/apps/git_source/output.txt b/acceptance/bundle/apps/git_source/output.txt index 6779647f8a..e9d7487bec 100644 --- a/acceptance/bundle/apps/git_source/output.txt +++ b/acceptance/bundle/apps/git_source/output.txt @@ -22,6 +22,16 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run my_app +✓ Getting the status of the app [APP_NAME] +✓ App is in RUNNING state +✓ App compute is in STOPPED state +✓ Starting the app [APP_NAME] +✓ App is starting... +✓ App is started! +✓ Deployment succeeded. +You can access the app at [APP_NAME]-123.cloud.databricksapps.com + === Get app details and verify git_source configuration >>> [CLI] bundle summary --output json @@ -45,7 +55,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ✓ Getting the status of the app [APP_NAME] ✓ App is in RUNNING state ✓ App compute is in ACTIVE state -✓ Deployment succeeded +✓ Deployment succeeded. You can access the app at [APP_NAME]-123.cloud.databricksapps.com === Update git_source branch and redeploy diff --git a/acceptance/bundle/apps/git_source/script b/acceptance/bundle/apps/git_source/script index b1241ec40a..962033f0e5 100644 --- a/acceptance/bundle/apps/git_source/script +++ b/acceptance/bundle/apps/git_source/script @@ -17,6 +17,7 @@ trace $CLI bundle plan title "Deploy bundle" trace $CLI bundle deploy +trace $CLI bundle run my_app > /dev/null || true title "Get app details and verify git_source configuration" app_name=$(trace $CLI bundle summary --output json | jq -r '.resources.apps.my_app.name') diff --git a/acceptance/bundle/apps/git_source/test.toml b/acceptance/bundle/apps/git_source/test.toml index 819e361dcd..db551625c7 100644 --- a/acceptance/bundle/apps/git_source/test.toml +++ b/acceptance/bundle/apps/git_source/test.toml @@ -9,42 +9,3 @@ Ignore = [".databricks", "databricks.yml", "databricks.yml.bak", "tmp.app-run"] # Apps can take longer to deploy TimeoutCloud = "5m" - -# Mock responses for app deployment -[[Server]] -Pattern = "POST /api/2.0/apps/{app_name}/deployments" -Response.Body = ''' -{ - "deployment_id": "test-deployment-123", - "status": { - "state": "SUCCEEDED", - "message": "Deployment succeeded" - }, - "git_source": { - "branch": "main", - "source_code_path": "internal/testdata/simple-app", - "resolved_commit": "abc123def456" - }, - "source_code_path": "/Workspace/Users/tester@databricks.com/.bundle/files/app", - "mode": "SNAPSHOT" -} -''' - -[[Server]] -Pattern = "GET /api/2.0/apps/{app_name}/deployments/{deployment_id}" -Response.Body = ''' -{ - "deployment_id": "test-deployment-123", - "status": { - "state": "SUCCEEDED", - "message": "Deployment succeeded" - }, - "git_source": { - "branch": "main", - "source_code_path": "internal/testdata/simple-app", - "resolved_commit": "abc123def456" - }, - "source_code_path": "/Workspace/Users/tester@databricks.com/.bundle/files/app", - "mode": "SNAPSHOT" -} -''' diff --git a/acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh b/acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh new file mode 100644 index 0000000000..a303e44c14 --- /dev/null +++ b/acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh @@ -0,0 +1,3 @@ +# Run the app after the deploy otherwise migrate will show the drift on the source code path. +# This happens because source code path is set remotely only after the deploy. +$CLI bundle run foo diff --git a/acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl b/acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl new file mode 100644 index 0000000000..f0f1e3cbc0 --- /dev/null +++ b/acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + apps: + foo: + name: app-foo-$UNIQUE_NAME + source_code_path: ./app + lifecycle: + started: true + bar: + name: app-bar-$UNIQUE_NAME + source_code_path: ./app + lifecycle: + started: ${resources.apps.foo.lifecycle.started} diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index d02200cb53..dba524bd8b 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -35,6 +35,10 @@ cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null echo INPUT_CONFIG_OK trace $CLI bundle deployment migrate &> LOG.migrate +POST_DEPLOY_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-post-deploy.sh" +if [ -f "$POST_DEPLOY_SCRIPT" ]; then + source "$POST_DEPLOY_SCRIPT" &> LOG.post_deploy +fi cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 0473672e16..2cf8ace75d 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -34,6 +34,10 @@ trap cleanup EXIT trace $CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null +POST_DEPLOY_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-post-deploy.sh" +if [ -f "$POST_DEPLOY_SCRIPT" ]; then + source "$POST_DEPLOY_SCRIPT" &> LOG.post_deploy +fi # Special message to fuzzer that generated config was fine. # Any failures after this point will be considered as "bug detected" by fuzzer. diff --git a/acceptance/bundle/lifecycle/started-validation/app/app.py b/acceptance/bundle/lifecycle/started-validation/app/app.py new file mode 100644 index 0000000000..d56323cf53 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/app/app.py @@ -0,0 +1 @@ +print("Hello world\!") diff --git a/acceptance/bundle/lifecycle/started-validation/databricks.yml b/acceptance/bundle/lifecycle/started-validation/databricks.yml new file mode 100644 index 0000000000..d6d2d81c2d --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test_lifecycle_started_validation + +resources: + jobs: + my_job: + name: my_job + lifecycle: + started: true + + apps: + my_app: + name: my_app + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/lifecycle/started-validation/out.direct.txt b/acceptance/bundle/lifecycle/started-validation/out.direct.txt new file mode 100644 index 0000000000..12a310841c --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/out.direct.txt @@ -0,0 +1,10 @@ + +>>> errcode [CLI] bundle plan +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 + +create apps.my_app +create jobs.my_job + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/lifecycle/started-validation/out.terraform.txt b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt new file mode 100644 index 0000000000..ef87f528c1 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt @@ -0,0 +1,11 @@ + +>>> errcode [CLI] bundle plan +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 + +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:16:18 + + +Exit code: 1 diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml new file mode 100644 index 0000000000..58724616cf --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started-validation/output.txt b/acceptance/bundle/lifecycle/started-validation/output.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/lifecycle/started-validation/script b/acceptance/bundle/lifecycle/started-validation/script new file mode 100644 index 0000000000..ff9d8c0454 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/script @@ -0,0 +1 @@ +trace errcode $CLI bundle plan >> out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 diff --git a/acceptance/bundle/lifecycle/started-validation/test.toml b/acceptance/bundle/lifecycle/started-validation/test.toml new file mode 100644 index 0000000000..bc76388306 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/test.toml @@ -0,0 +1,4 @@ +Ignore = [".databricks"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started/databricks.yml b/acceptance/bundle/lifecycle/started/databricks.yml new file mode 100644 index 0000000000..313b1cb8de --- /dev/null +++ b/acceptance/bundle/lifecycle/started/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test_lifecycle_started + +resources: + jobs: + my_job: + name: my_job + lifecycle: + started: true diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/lifecycle/started/output.txt b/acceptance/bundle/lifecycle/started/output.txt new file mode 100644 index 0000000000..0b4912bfe3 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/output.txt @@ -0,0 +1,7 @@ +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 + +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/lifecycle/started/script b/acceptance/bundle/lifecycle/started/script new file mode 100644 index 0000000000..4fbc2b517c --- /dev/null +++ b/acceptance/bundle/lifecycle/started/script @@ -0,0 +1 @@ +errcode $CLI bundle plan diff --git a/acceptance/bundle/lifecycle/started/test.toml b/acceptance/bundle/lifecycle/started/test.toml new file mode 100644 index 0000000000..d1240963e0 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/test.toml @@ -0,0 +1,7 @@ +Local = true +Cloud = false + +Ignore = [".databricks"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d032aa48f8..ecfab1562e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -93,14 +93,14 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.active_instances int ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL -resources.apps.*.config *resources.AppConfig INPUT -resources.apps.*.config.command []string INPUT -resources.apps.*.config.command[*] string INPUT -resources.apps.*.config.env []resources.AppEnvVar INPUT -resources.apps.*.config.env[*] resources.AppEnvVar INPUT -resources.apps.*.config.env[*].name string INPUT -resources.apps.*.config.env[*].value string INPUT -resources.apps.*.config.env[*].value_from string INPUT +resources.apps.*.config *resources.AppConfig ALL +resources.apps.*.config.command []string ALL +resources.apps.*.config.command[*] string ALL +resources.apps.*.config.env []resources.AppEnvVar ALL +resources.apps.*.config.env[*] resources.AppEnvVar ALL +resources.apps.*.config.env[*].name string ALL +resources.apps.*.config.env[*].value string ALL +resources.apps.*.config.env[*].value_from string ALL resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -112,18 +112,21 @@ resources.apps.*.effective_user_api_scopes[*] string ALL resources.apps.*.git_repository *apps.GitRepository ALL resources.apps.*.git_repository.provider string ALL resources.apps.*.git_repository.url string ALL -resources.apps.*.git_source *apps.GitSource INPUT -resources.apps.*.git_source.branch string INPUT -resources.apps.*.git_source.commit string INPUT -resources.apps.*.git_source.git_repository *apps.GitRepository INPUT -resources.apps.*.git_source.git_repository.provider string INPUT -resources.apps.*.git_source.git_repository.url string INPUT -resources.apps.*.git_source.resolved_commit string INPUT -resources.apps.*.git_source.source_code_path string INPUT -resources.apps.*.git_source.tag string INPUT +resources.apps.*.git_source *apps.GitSource ALL +resources.apps.*.git_source.branch string ALL +resources.apps.*.git_source.commit string ALL +resources.apps.*.git_source.git_repository *apps.GitRepository ALL +resources.apps.*.git_source.git_repository.provider string ALL +resources.apps.*.git_source.git_repository.url string ALL +resources.apps.*.git_source.resolved_commit string ALL +resources.apps.*.git_source.source_code_path string ALL +resources.apps.*.git_source.tag string ALL resources.apps.*.id string ALL +resources.apps.*.lifecycle *dresources.AppStateLifecycle REMOTE STATE +resources.apps.*.lifecycle *resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT +resources.apps.*.lifecycle.started *bool ALL resources.apps.*.modified_status string INPUT resources.apps.*.name string ALL resources.apps.*.oauth2_app_client_id string ALL @@ -197,7 +200,7 @@ resources.apps.*.resources[*].uc_securable.securable_type apps.AppResourceUcSecu resources.apps.*.service_principal_client_id string ALL resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL -resources.apps.*.source_code_path string INPUT +resources.apps.*.source_code_path string ALL resources.apps.*.space string ALL resources.apps.*.telemetry_export_destinations []apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*] apps.TelemetryExportDestination ALL diff --git a/acceptance/bundle/resources/apps/config-drift/app/app.py b/acceptance/bundle/resources/apps/config-drift/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl b/acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl new file mode 100644 index 0000000000..b38d8f92f5 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: config-drift-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app + source_code_path: ./app + config: + command: + - python + - app.py + env: + - name: MY_VAR + value: original_value + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json b/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json new file mode 100644 index 0000000000..e3571f45fa --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json @@ -0,0 +1,123 @@ +{ + "active_deployment": { + "action": "skip", + "reason": "spec:output_only", + "remote": { + "command": [ + "streamlit", + "run", + "dashboard.py" + ], + "deployment_id": "deploy-[NUMID]", + "env_vars": [ + { + "name": "MY_VAR", + "value": "changed_value" + }, + { + "name": "NEW_VAR", + "value": "new_value" + } + ], + "mode": "SNAPSHOT", + "source_code_path": "./app", + "status": { + "message": "Deployment succeeded.", + "state": "SUCCEEDED" + } + } + }, + "app_status": { + "action": "skip", + "reason": "spec:output_only", + "remote": { + "message": "Application is running.", + "state": "RUNNING" + } + }, + "compute_status": { + "action": "skip", + "reason": "spec:output_only", + "remote": { + "message": "App compute is active.", + "state": "ACTIVE" + } + }, + "config.command": { + "action": "update", + "old": [ + "python", + "app.py" + ], + "new": [ + "python", + "app.py" + ], + "remote": [ + "streamlit", + "run", + "dashboard.py" + ] + }, + "config.env": { + "action": "update", + "old": [ + { + "name": "MY_VAR", + "value": "original_value" + } + ], + "new": [ + { + "name": "MY_VAR", + "value": "original_value" + } + ], + "remote": [ + { + "name": "MY_VAR", + "value": "changed_value" + }, + { + "name": "NEW_VAR", + "value": "new_value" + } + ] + }, + "default_source_code_path": { + "action": "skip", + "reason": "spec:output_only", + "remote": "./app" + }, + "id": { + "action": "skip", + "reason": "spec:output_only", + "remote": "1000" + }, + "service_principal_client_id": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[UUID]" + }, + "service_principal_id": { + "action": "skip", + "reason": "spec:output_only", + "remote": [NUMID] + }, + "service_principal_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "app-[UNIQUE_NAME]" + }, + "source_code_path": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app", + "new": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app", + "remote": "./app" + }, + "url": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[UNIQUE_NAME]-123.cloud.databricksapps.com" + } +} diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift/output.txt b/acceptance/bundle/resources/apps/config-drift/output.txt new file mode 100644 index 0000000000..1965decd17 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/output.txt @@ -0,0 +1,54 @@ + +=== First deploy: creates app +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Second deploy: pushes code with config +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify no drift after deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Simulate out-of-band deployment with changed command and env +=== Plan should detect config drift +>>> [CLI] bundle plan +update apps.myapp + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +=== Redeploy to fix drift +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify no drift after fix +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Simulate out-of-band deployment with git_source added +=== Plan should detect git_source drift +>>> [CLI] bundle plan +update apps.myapp + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/config-drift/script b/acceptance/bundle/resources/apps/config-drift/script new file mode 100644 index 0000000000..e37ac80fde --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/script @@ -0,0 +1,49 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "First deploy: creates app" +trace $CLI bundle deploy + +title "Second deploy: pushes code with config" +trace $CLI bundle deploy + +title "Verify no drift after deploy" +trace $CLI bundle plan + +title "Simulate out-of-band deployment with changed command and env" +$CLI apps deploy $UNIQUE_NAME --no-wait --json '{ + "source_code_path": "./app", + "mode": "SNAPSHOT", + "command": ["streamlit", "run", "dashboard.py"], + "env_vars": [ + {"name": "MY_VAR", "value": "changed_value"}, + {"name": "NEW_VAR", "value": "new_value"} + ] + }' > /dev/null + +title "Plan should detect config drift" +trace $CLI bundle plan +$CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' > out.plan.direct.json + +title "Redeploy to fix drift" +trace $CLI bundle deploy + +title "Verify no drift after fix" +trace $CLI bundle plan + +title "Simulate out-of-band deployment with git_source added" +$CLI apps deploy $UNIQUE_NAME --no-wait --json '{ + "source_code_path": "./app", + "mode": "SNAPSHOT", + "git_source": {"branch": "feature-branch"}, + "command": ["python", "app.py"], + "env_vars": [{"name": "MY_VAR", "value": "original_value"}] + }' > /dev/null + +title "Plan should detect git_source drift" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/apps/config-drift/test.toml b/acceptance/bundle/resources/apps/config-drift/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py new file mode 100644 index 0000000000..d56323cf53 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py @@ -0,0 +1 @@ +print("Hello world\!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl new file mode 100644 index 0000000000..ca2eb952d0 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: lifecycle-started-omitted-$UNIQUE_NAME + +resources: + apps: + mykey: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt new file mode 100644 index 0000000000..8f5e0b61bf --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt @@ -0,0 +1,81 @@ + +=== (started omitted) -> deploy: app created with no_compute=true, no start/stop requests +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + } +} + +=== (started omitted) -> started: false -> deploy: no start/stop requests +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests + +=== started: false -> (started omitted) -> deploy: no start/stop requests +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests + +=== (started omitted) -> started: true -> deploy: app started +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments" +} + +=== started: true -> (started omitted) -> deploy: no start/stop requests (compute stays as-is) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests + +=== (started omitted, app running) -> bundle plan shows no driftPlan: no drift detected + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.mykey + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script new file mode 100644 index 0000000000..ca97563ffa --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -0,0 +1,85 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_app_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps")))' < out.requests.txt + rm out.requests.txt +} + +title "(started omitted) -> deploy: app created with no_compute=true, no start/stop requests" +trace $CLI bundle deploy +trace print_app_requests + +title "(started omitted) -> started: false -> deploy: no start/stop requests" +cat > databricks.yml < (started omitted) -> deploy: no start/stop requests" +cat > databricks.yml < started: true -> deploy: app started" +cat > databricks.yml < (started omitted) -> deploy: no start/stop requests (compute stays as-is)" +cat > databricks.yml < bundle plan shows no drift" +$CLI bundle plan -o json > LOG.planjson +verify_no_drift.py LOG.planjson +echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl new file mode 100644 index 0000000000..ee78dfc150 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml new file mode 100644 index 0000000000..a9f28de48a --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt new file mode 100644 index 0000000000..80395fe51a --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt @@ -0,0 +1,16 @@ + +=== bundle plan fails with lifecycle.started on terraform engine +>>> errcode [CLI] bundle plan +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 + + +Exit code: 1 + +=== bundle deploy fails with lifecycle.started on terraform engine +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script new file mode 100644 index 0000000000..4fb2038cd2 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script @@ -0,0 +1,7 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "bundle plan fails with lifecycle.started on terraform engine" +trace errcode $CLI bundle plan + +title "bundle deploy fails with lifecycle.started on terraform engine" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml new file mode 100644 index 0000000000..34b19e94a9 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RecordRequests = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py new file mode 100644 index 0000000000..d56323cf53 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py @@ -0,0 +1 @@ +print("Hello world\!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl new file mode 100644 index 0000000000..b6ecf37530 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-toggle-$UNIQUE_NAME + +resources: + apps: + mykey: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: false diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt new file mode 100644 index 0000000000..d1d37d7b53 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -0,0 +1,79 @@ + +=== Deploy with started=false: app created without compute (no_compute=true) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +=== Toggle started=false -> started=true: only Start should be called, no Update +>>> update_file.py databricks.yml started: false started: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments" +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +=== Toggle started=true -> started=false: only Stop should be called, no Update +>>> update_file.py databricks.yml started: true started: false + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/stop" +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.mykey + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/script b/acceptance/bundle/resources/apps/lifecycle-started-toggle/script new file mode 100644 index 0000000000..7b5a2c8c32 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_app_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps"))) | del(.body.url)' < out.requests.txt + rm out.requests.txt +} + +title "Deploy with started=false: app created without compute (no_compute=true)" +trace $CLI bundle deploy +trace print_app_requests +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true + +title "Toggle started=false -> started=true: only Start should be called, no Update" +trace update_file.py databricks.yml "started: false" "started: true" +trace $CLI bundle deploy +trace print_app_requests +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true + +title "Toggle started=true -> started=false: only Stop should be called, no Update" +trace update_file.py databricks.yml "started: true" "started: false" +trace $CLI bundle deploy +trace print_app_requests +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl new file mode 100644 index 0000000000..ee78dfc150 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt new file mode 100644 index 0000000000..1ba62d6df0 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -0,0 +1,98 @@ + +=== Deploy with started=true: app created with compute running +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps", + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +=== Re-deploy with description change: code deployed again +>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +=== Stop app externally, then deploy with started=false: app stays stopped +>>> errcode [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + +>>> update_file.py databricks.yml started: true started: false + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +=== Deploy with started=true: compute restarted and code deployed +>>> update_file.py databricks.yml started: false started: true + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script new file mode 100644 index 0000000000..53d2bf5773 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -0,0 +1,36 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy with started=true: app created with compute running" +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt; } || true +rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true + +title "Re-deploy with description change: code deployed again" +trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } || true +rm -f out.requests.txt + +title "Stop app externally, then deploy with started=false: app stays stopped" +{ trace errcode $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state'; } || true +trace update_file.py databricks.yml "started: true" "started: false" +trace update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } || true +rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true + +title "Deploy with started=true: compute restarted and code deployed" +trace update_file.py databricks.yml "started: false" "started: true" +trace update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt; } || true +rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/test.toml b/acceptance/bundle/resources/apps/lifecycle-started/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/update/out.requests.direct.json b/acceptance/bundle/resources/apps/update/out.requests.direct.json index 85a9ac2bc6..8bf3fbbf82 100644 --- a/acceptance/bundle/resources/apps/update/out.requests.direct.json +++ b/acceptance/bundle/resources/apps/update/out.requests.direct.json @@ -9,6 +9,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/myappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": { "app": { @@ -20,6 +33,14 @@ "method": "POST", "path": "/api/2.0/apps/myappname/update" } +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": {}, "method": "DELETE", @@ -36,6 +57,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/deployments" +} { "body": {}, "method": "DELETE", diff --git a/acceptance/bundle/resources/apps/update/out.requests.terraform.json b/acceptance/bundle/resources/apps/update/out.requests.terraform.json index 98dfdcccdc..253def8245 100644 --- a/acceptance/bundle/resources/apps/update/out.requests.terraform.json +++ b/acceptance/bundle/resources/apps/update/out.requests.terraform.json @@ -9,6 +9,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/myappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": { "description": "MY_APP_DESCRIPTION", @@ -20,6 +33,14 @@ "method": "PATCH", "path": "/api/2.0/apps/myappname" } +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": {}, "method": "DELETE", @@ -36,6 +57,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/deployments" +} { "body": {}, "method": "DELETE", diff --git a/acceptance/bundle/resources/apps/update/output.txt b/acceptance/bundle/resources/apps/update/output.txt index 021b10a9cd..5b94b6d4af 100644 --- a/acceptance/bundle/resources/apps/update/output.txt +++ b/acceptance/bundle/resources/apps/update/output.txt @@ -10,6 +10,16 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run mykey +✓ Getting the status of the app myappname +✓ App is in RUNNING state +✓ App compute is in STOPPED state +✓ Starting the app myappname +✓ App is starting... +✓ App is started! +✓ Deployment succeeded. +You can access the app at myappname-123.cloud.databricksapps.com + >>> print_requests >>> [CLI] bundle summary @@ -38,6 +48,13 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run mykey +✓ Getting the status of the app myappname +✓ App is in RUNNING state +✓ App compute is in ACTIVE state +✓ Deployment succeeded. +You can access the app at myappname-123.cloud.databricksapps.com + >>> print_requests >>> [CLI] bundle summary @@ -78,6 +95,16 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run mykey +✓ Getting the status of the app mynewappname +✓ App is in RUNNING state +✓ App compute is in STOPPED state +✓ Starting the app mynewappname +✓ App is starting... +✓ App is started! +✓ Deployment succeeded. +You can access the app at mynewappname-123.cloud.databricksapps.com + >>> print_requests >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/apps/update/script b/acceptance/bundle/resources/apps/update/script index a66ff34b8c..22e832b035 100644 --- a/acceptance/bundle/resources/apps/update/script +++ b/acceptance/bundle/resources/apps/update/script @@ -6,6 +6,7 @@ print_requests() { trace $CLI bundle plan trace $CLI bundle deploy +trace $CLI bundle run mykey trace print_requests > out.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle summary @@ -13,6 +14,7 @@ title "Update description and re-deploy" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION trace $CLI bundle plan trace $CLI bundle deploy +trace $CLI bundle run mykey trace print_requests >> out.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle summary @@ -21,6 +23,7 @@ trace update_file.py databricks.yml myappname mynewappname trace $CLI bundle plan trace $CLI bundle summary trace $CLI bundle deploy +trace $CLI bundle run mykey trace print_requests >> out.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json index a970eafec7..40c12a2c54 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json @@ -7,7 +7,8 @@ "new_state": { "value": { "description": "", - "name": "foo" + "name": "foo", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } }, diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json index a970eafec7..40c12a2c54 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json @@ -7,7 +7,8 @@ "new_state": { "value": { "description": "", - "name": "foo" + "name": "foo", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } }, diff --git a/bundle/appdeploy/app.go b/bundle/appdeploy/app.go new file mode 100644 index 0000000000..e73922d490 --- /dev/null +++ b/bundle/appdeploy/app.go @@ -0,0 +1,110 @@ +package appdeploy + +import ( + "context" + "fmt" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" + sdkapps "github.com/databricks/databricks-sdk-go/service/apps" +) + +func logProgress(ctx context.Context, msg string) { + if msg == "" { + return + } + cmdio.LogString(ctx, "✓ "+msg) +} + +// BuildDeployment constructs an AppDeployment from the app's source code path, inline config and git source. +func BuildDeployment(sourcePath string, config *resources.AppConfig, gitSource *sdkapps.GitSource) sdkapps.AppDeployment { + deployment := sdkapps.AppDeployment{ + Mode: sdkapps.AppDeploymentModeSnapshot, + SourceCodePath: sourcePath, + } + + if gitSource != nil { + deployment.GitSource = gitSource + } + + if config != nil { + if len(config.Command) > 0 { + deployment.Command = config.Command + } + + if len(config.Env) > 0 { + deployment.EnvVars = make([]sdkapps.EnvVar, len(config.Env)) + for i, env := range config.Env { + deployment.EnvVars[i] = sdkapps.EnvVar{ + Name: env.Name, + Value: env.Value, + ValueFrom: env.ValueFrom, + } + } + } + } + + return deployment +} + +// WaitForDeploymentToComplete waits for active and pending deployments on an app to finish. +func WaitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *sdkapps.App) error { + if app.ActiveDeployment != nil && + app.ActiveDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the active deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Active deployment is completed!") + } + + if app.PendingDeployment != nil && + app.PendingDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the pending deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Pending deployment is completed!") + } + + return nil +} + +// Deploy deploys the app using the provided deployment request. +// If another deployment is in progress, it waits for it to complete and retries. +func Deploy(ctx context.Context, w *databricks.WorkspaceClient, appName string, deployment sdkapps.AppDeployment) error { + wait, err := w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + existingApp, getErr := w.Apps.Get(ctx, sdkapps.GetAppRequest{Name: appName}) + if getErr != nil { + return fmt.Errorf("failed to get app %s: %w", appName, getErr) + } + + if waitErr := WaitForDeploymentToComplete(ctx, w, existingApp); waitErr != nil { + return waitErr + } + + wait, err = w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + return err + } + } + + _, err = wait.OnProgress(func(ad *sdkapps.AppDeployment) { + if ad.Status == nil { + return + } + logProgress(ctx, ad.Status.Message) + }).Get() + return err +} diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go new file mode 100644 index 0000000000..ee88e413c0 --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -0,0 +1,50 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +type validateLifecycleStarted struct { + engine engine.EngineType +} + +// ValidateLifecycleStarted returns a mutator that errors when lifecycle.started +// is used with the terraform deployment engine. +// lifecycle.started is only supported in direct deployment mode. +func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { + return &validateLifecycleStarted{engine: e} +} + +func (m *validateLifecycleStarted) Name() string { + return "ValidateLifecycleStarted" +} + +func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + for _, group := range b.Config.Resources.AllResources() { + for key, resource := range group.Resources { + lws, ok := resource.GetLifecycle().(resources.LifecycleWithStarted) + if !ok || lws.Started == nil { + continue + } + + // lifecycle.started is a direct-mode-only feature. + if !m.engine.IsDirect() { + path := "resources." + group.Description.PluralName + "." + key + ".lifecycle.started" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.started is only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), + }) + } + } + } + + return diags +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index aaf4687a84..4131f68695 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -53,6 +53,9 @@ type ConfigResource interface { // InitializeURL initializes the URL field of the resource. InitializeURL(baseURL url.URL) + + // GetLifecycle returns the lifecycle settings for the resource. + GetLifecycle() resources.LifecycleConfig } // ResourceGroup represents a group of resources of the same type. diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go index e48f6e7dae..f17aa4c22b 100644 --- a/bundle/config/resources/apps.go +++ b/bundle/config/resources/apps.go @@ -37,6 +37,9 @@ type App struct { apps.App // nolint App struct also defines Id and URL field with the same json tag "id" and "url" // Note: apps.App already includes GitRepository field from the SDK + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"` + // SourceCodePath is a required field used by DABs to point to Databricks app source code // on local disk and to the corresponding workspace path during app deployment. SourceCodePath string `json:"source_code_path,omitempty"` @@ -53,6 +56,14 @@ type App struct { Permissions []AppPermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (a *App) GetLifecycle() LifecycleConfig { + if a.Lifecycle == nil { + return LifecycleWithStarted{} + } + return *a.Lifecycle +} + func (a *App) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, a) } diff --git a/bundle/config/resources/base.go b/bundle/config/resources/base.go index 792db28972..ceeb1f0b86 100644 --- a/bundle/config/resources/base.go +++ b/bundle/config/resources/base.go @@ -7,3 +7,8 @@ type BaseResource struct { URL string `json:"url,omitempty" bundle:"internal"` Lifecycle Lifecycle `json:"lifecycle,omitempty"` } + +// GetLifecycle returns the lifecycle settings for the resource. +func (b *BaseResource) GetLifecycle() LifecycleConfig { + return b.Lifecycle +} diff --git a/bundle/config/resources/external_location.go b/bundle/config/resources/external_location.go index 29cb5469d2..cc413de84f 100644 --- a/bundle/config/resources/external_location.go +++ b/bundle/config/resources/external_location.go @@ -62,6 +62,11 @@ func (e *ExternalLocation) GetName() string { return e.Name } +// GetLifecycle returns the lifecycle settings for the resource. +func (e *ExternalLocation) GetLifecycle() LifecycleConfig { + return e.Lifecycle +} + func (e *ExternalLocation) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, e) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index c3de7ce8ea..db2b130d31 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -1,8 +1,27 @@ package resources -// Lifecycle is a struct that contains the lifecycle settings for a resource. -// It controls the behavior of the resource when it is deployed or destroyed. +// LifecycleConfig is implemented by Lifecycle and LifecycleWithStarted. +type LifecycleConfig interface { + HasPreventDestroy() bool +} + +// Lifecycle contains base lifecycle settings supported by all resources. type Lifecycle struct { // Lifecycle setting to prevent the resource from being destroyed. PreventDestroy bool `json:"prevent_destroy,omitempty"` } + +// HasPreventDestroy returns true if prevent_destroy is set. +func (l Lifecycle) HasPreventDestroy() bool { + return l.PreventDestroy +} + +// LifecycleWithStarted contains lifecycle settings for resources that support lifecycle.started. +// It is used by apps, clusters, and sql_warehouses. +type LifecycleWithStarted struct { + Lifecycle + + // If set to true, the resource will be deployed in started mode. + // Supported only for apps, clusters, and sql_warehouses. + Started *bool `json:"started,omitempty"` +} diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go index b25d403766..87095eb0aa 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app.go +++ b/bundle/deploy/terraform/tfdyn/convert_app.go @@ -38,7 +38,7 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out return err } - // We always set no_compute to true as it allows DABs not to wait for app compute to be started when app is created. + // Always skip compute during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_compute", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/lifecycle.go b/bundle/deploy/terraform/tfdyn/lifecycle.go index 1600cdef2d..669aa83040 100644 --- a/bundle/deploy/terraform/tfdyn/lifecycle.go +++ b/bundle/deploy/terraform/tfdyn/lifecycle.go @@ -11,7 +11,19 @@ func convertLifecycle(ctx context.Context, vout, vLifecycle dyn.Value) (dyn.Valu return vout, nil } - vout, err := dyn.Set(vout, "lifecycle", vLifecycle) + // Strip lifecycle.started: it is a DABs-only field not understood by Terraform. + var err error + vLifecycle, err = dyn.DropKeys(vLifecycle, []string{"started"}) + if err != nil { + return dyn.InvalidValue, err + } + + // If only lifecycle.started was set (now empty), skip setting the lifecycle block. + if m, ok := vLifecycle.AsMap(); ok && m.Len() == 0 { + return vout, nil + } + + vout, err = dyn.Set(vout, "lifecycle", vLifecycle) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 1a5188c335..3f4b4f7401 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -4,16 +4,63 @@ import ( "context" "errors" "fmt" + "slices" "strings" "time" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/apps" ) +// AppStateLifecycle holds lifecycle settings persisted in state. +type AppStateLifecycle struct { + Started *bool `json:"started,omitempty"` +} + +// AppState is the state type for App resources. It extends apps.App with deployment-related +// fields (source_code_path, config, git_source, lifecycle) that are persisted in state. +type AppState struct { + apps.App + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` +} + +// AppRemote extends apps.App with the same deployment fields as AppState so they +// appear in RemoteType and can be used for $resource resolution and drift detection. +type AppRemote struct { + apps.App + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` +} + +// Custom marshalers needed because embedded apps.App has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *AppState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s AppState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (r *AppRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, r) +} + +func (r AppRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(r) +} + type ResourceApp struct { client *databricks.WorkspaceClient } @@ -22,18 +69,65 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { return &ResourceApp{client: client} } -func (*ResourceApp) PrepareState(input *resources.App) *apps.App { - return &input.App +func (*ResourceApp) PrepareState(input *resources.App) *AppState { + s := &AppState{ + App: input.App, + SourceCodePath: input.SourceCodePath, + Config: input.Config, + GitSource: input.GitSource, + Lifecycle: nil, + } + if input.Lifecycle != nil && input.Lifecycle.Started != nil { + s.Lifecycle = &AppStateLifecycle{Started: input.Lifecycle.Started} + } + return s +} + +// RemapState maps the remote AppRemote to AppState for diff comparison. +// Config, GitSource, and SourceCodePath are populated from the active deployment +// when one exists, enabling drift detection for out-of-band redeploys. +// Started is derived from compute status so the planner can detect start/stop changes. +func (*ResourceApp) RemapState(remote *AppRemote) *AppState { + started := !isComputeStopped(&remote.App) + return &AppState{ + App: remote.App, + SourceCodePath: remote.SourceCodePath, + Config: remote.Config, + GitSource: remote.GitSource, + Lifecycle: &AppStateLifecycle{Started: &started}, + } } -func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) { - return r.client.Apps.GetByName(ctx, id) +func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) { + app, err := r.client.Apps.GetByName(ctx, id) + if err != nil { + return nil, err + } + started := !isComputeStopped(app) + remote := &AppRemote{ + App: *app, + Config: nil, + GitSource: nil, + SourceCodePath: "", + Lifecycle: &AppStateLifecycle{Started: &started}, + } + if app.ActiveDeployment != nil { + // The source code path in active deployment is snapshotted version of the source code path in the app. + // We need to use the default source code path to get the correct source code path for drift detection. + remote.SourceCodePath = app.DefaultSourceCodePath + remote.GitSource = app.ActiveDeployment.GitSource + remote.Config = deploymentToAppConfig(app.ActiveDeployment) + } + return remote, nil } -func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, *apps.App, error) { +func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *AppRemote, error) { + // Start app compute only when lifecycle.started=true is explicit. + // For nil (omitted) or false, use no_compute=true (do not start compute). + noCompute := config.Lifecycle == nil || config.Lifecycle.Started == nil || !*config.Lifecycle.Started request := apps.CreateAppRequest{ - App: *config, - NoCompute: true, + App: config.App, + NoCompute: noCompute, ForceSendFields: nil, } @@ -68,36 +162,148 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, * return app.Name, nil, nil } -func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, entry *PlanEntry) (*apps.App, error) { - updateMask := strings.Join(collectUpdatePathsWithPrefix(entry.Changes, ""), ",") - - request := apps.AsyncUpdateAppRequest{ - App: config, - AppName: id, - UpdateMask: updateMask, +func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) (*AppRemote, error) { + // Build update mask excluding local-only fields that have no counterpart in the App Update API. + // Paths are truncated at the first index because the API only supports entire collection fields, + // not individual elements (e.g. "resources" instead of "resources[0].name"). + maskSet := make(map[string]bool) + for path, change := range entry.Changes { + if change.Action != deployplan.Update { + continue + } + truncated := truncateAtIndex(path) + if !deployOnlyFields[truncated] { + maskSet[truncated] = true + } } - updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) - if err != nil { - return nil, err + maskPaths := make([]string, 0, len(maskSet)) + for p := range maskSet { + maskPaths = append(maskPaths, p) } + slices.Sort(maskPaths) + updateMask := strings.Join(maskPaths, ",") - response, err := updateWaiter.Get() - if err != nil { - return nil, err + if updateMask != "" { + request := apps.AsyncUpdateAppRequest{ + App: &config.App, + AppName: id, + UpdateMask: updateMask, + } + updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) + if err != nil { + return nil, err + } + + response, err := updateWaiter.Get() + if err != nil { + return nil, err + } + + if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { + return nil, fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message) + } + } + + if config.Lifecycle == nil || config.Lifecycle.Started == nil { + return nil, nil } - if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { - return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) + desiredStarted := *config.Lifecycle.Started + remoteStarted := remoteIsStarted(entry) + + if desiredStarted { + // lifecycle.started=true: ensure the app compute is running and deploy the latest code. + if !remoteStarted { + startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) + if err != nil { + return nil, err + } + startedApp, err := startWaiter.Get() + if err != nil { + return nil, err + } + if err := appdeploy.WaitForDeploymentToComplete(ctx, r.client, startedApp); err != nil { + return nil, err + } + } + deployment := appdeploy.BuildDeployment(config.SourceCodePath, config.Config, config.GitSource) + if err := appdeploy.Deploy(ctx, r.client, id, deployment); err != nil { + return nil, err + } + } else { + // lifecycle.started=false: ensure the app compute is stopped. + if remoteStarted { + stopWaiter, err := r.client.Apps.Stop(ctx, apps.StopAppRequest{Name: id}) + if err != nil { + return nil, err + } + if _, err = stopWaiter.Get(); err != nil { + return nil, err + } + } } + return nil, nil } +// deployOnlyFields are AppState fields managed via the Deploy API, not the App Update API. +// They have remote counterparts (populated from active deployment and compute status), +// but must not appear in the App update_mask. +var deployOnlyFields = map[string]bool{ + "source_code_path": true, + "config": true, + "git_source": true, + "lifecycle": true, + "lifecycle.started": true, +} + +func isComputeStopped(app *apps.App) bool { + return app.ComputeStatus == nil || + app.ComputeStatus.State == apps.ComputeStateStopped || + app.ComputeStatus.State == apps.ComputeStateError +} + +// remoteIsStarted reads the compute started state from the plan entry's remote state. +func remoteIsStarted(entry *PlanEntry) bool { + if entry.RemoteState == nil { + return false + } + remote, ok := entry.RemoteState.(*AppRemote) + if !ok || remote.Lifecycle == nil || remote.Lifecycle.Started == nil { + return false + } + return *remote.Lifecycle.Started +} + +// deploymentToAppConfig extracts an AppConfig from an active deployment. +// Returns nil if the deployment has no command or env vars. +func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { + if len(d.Command) == 0 && len(d.EnvVars) == 0 { + return nil + } + config := &resources.AppConfig{ + Command: d.Command, + Env: nil, + } + if len(d.EnvVars) > 0 { + config.Env = make([]resources.AppEnvVar, len(d.EnvVars)) + for i, ev := range d.EnvVars { + config.Env[i] = resources.AppEnvVar{ + Name: ev.Name, + Value: ev.Value, + ValueFrom: ev.ValueFrom, + } + } + } + return config +} + func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { _, err := r.client.Apps.DeleteByName(ctx, id) return err } -func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *apps.App) (*apps.App, error) { +func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*AppRemote, error) { return r.waitForApp(ctx, r.client, config.Name) } @@ -106,9 +312,9 @@ func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *apps.App) (*a // We can't use the default waiter from SDK because it only waits on ACTIVE state but we need also STOPPED state. // Ideally this should be done in Go SDK but currently only ACTIVE is marked as terminal state // so this would need to be addressed by Apps service team first in their proto. -func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceClient, name string) (*apps.App, error) { +func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceClient, name string) (*AppRemote, error) { retrier := retries.New[apps.App](retries.WithTimeout(-1), retries.WithRetryFunc(shouldRetry)) - return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { + app, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { app, err := w.Apps.GetByName(ctx, name) if err != nil { return nil, retries.Halt(err) @@ -126,4 +332,21 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli return nil, retries.Continues(statusMessage) } }) + if err != nil { + return nil, err + } + started := !isComputeStopped(app) + remote := &AppRemote{ + App: *app, + Config: nil, + GitSource: nil, + SourceCodePath: "", + Lifecycle: &AppStateLifecycle{Started: &started}, + } + if app.ActiveDeployment != nil { + remote.SourceCodePath = app.DefaultSourceCodePath + remote.GitSource = app.ActiveDeployment.GitSource + remote.Config = deploymentToAppConfig(app.ActiveDeployment) + } + return remote, nil } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index e0eeed5b77..5fef3e45c9 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -1,8 +1,10 @@ package dresources import ( + "encoding/json" "testing" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" @@ -10,6 +12,37 @@ import ( "github.com/stretchr/testify/require" ) +// TestAppStateMarshalUnmarshal verifies that AppState correctly preserves bundle-only fields +// (SourceCodePath, Config, GitSource, Lifecycle) through a JSON round-trip. +// Without the custom marshaler, apps.App's promoted MarshalJSON would drop these extra fields. +func TestAppStateMarshalUnmarshal(t *testing.T) { + started := true + original := AppState{ + App: apps.App{ + Name: "my-app", + Description: "test description", + }, + SourceCodePath: "/Workspace/Users/user/.bundle/app/files", + Config: &resources.AppConfig{ + Command: []string{"python", "app.py"}, + }, + Lifecycle: &AppStateLifecycle{Started: &started}, + } + + data, err := json.Marshal(original) + require.NoError(t, err) + + var restored AppState + require.NoError(t, json.Unmarshal(data, &restored)) + + assert.Equal(t, original.Name, restored.Name) + assert.Equal(t, original.Description, restored.Description) + assert.Equal(t, original.SourceCodePath, restored.SourceCodePath) + assert.Equal(t, original.Config, restored.Config) + require.NotNil(t, restored.Lifecycle) + assert.Equal(t, original.Lifecycle.Started, restored.Lifecycle.Started) +} + // TestAppDoCreate_RetriesWhenAppIsDeleting verifies that DoCreate retries when // an app already exists but is in DELETING state. func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) { @@ -57,7 +90,7 @@ func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) @@ -113,7 +146,7 @@ func TestAppDoCreate_RetriesWhenGetReturnsNotFound(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index eea56c90e8..a58022aa70 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -375,10 +375,40 @@ resources: recreate_on_changes: - field: name reason: immutable + ignore_remote_changes: + # lifecycle fields are managed by the planner based on compute status. + # When lifecycle.started is omitted from config, it should be untracked + # and remote changes to compute state should not cause drift. + - field: lifecycle + reason: local_only + - field: lifecycle.started + reason: local_only backend_defaults: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/internal/providers/pluginfw/products/app/resource_app.go#L41 # s["compute_size"] = s["compute_size"].SetComputed() - field: compute_size + - field: app_status + reason: output_only + - field: compute_status + reason: output_only + - field: url + reason: output_only + - field: service_principal_client_id + reason: output_only + - field: service_principal_id + reason: output_only + - field: service_principal_name + reason: output_only + - field: create_time + reason: output_only + - field: update_time + reason: output_only + - field: creator + reason: output_only + - field: updater + reason: output_only + - field: effective_budget_policy_id + reason: output_only secret_scopes: backend_defaults: diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d061d4d0da..3321725049 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -92,9 +92,7 @@ var knownMissingInStateType = map[string][]string{ "file_path", }, "apps": { - "config", - "source_code_path", - "git_source", + "lifecycle.prevent_destroy", }, "dashboards": { "file_path", diff --git a/bundle/direct/dresources/util.go b/bundle/direct/dresources/util.go index 5a7671e8bc..66b3ef243d 100644 --- a/bundle/direct/dresources/util.go +++ b/bundle/direct/dresources/util.go @@ -61,3 +61,17 @@ func collectUpdatePathsWithPrefix(changes Changes, prefix string) []string { } return paths } + +// truncateAtIndex truncates a field path at the first bracket index (e.g. "[0]", "[*]", +// "[key=value]"). Most update_mask APIs only support referencing entire collection +// fields, not individual elements within them. +// Examples: "resources[0].name" -> "resources", "description" -> "description", +// "config.env[0].name" -> "config.env". +func truncateAtIndex(path string) string { + for i, c := range path { + if c == '[' { + return path[:i] + } + } + return path +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f30e054ed8..459d2c19f6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -661,6 +661,13 @@ github.com/databricks/cli/bundle/config/resources.Lifecycle: "prevent_destroy": "description": |- Lifecycle setting to prevent the resource from being destroyed. +github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted: + "prevent_destroy": + "description": |- + Lifecycle setting to prevent the resource from being destroyed. + "started": + "description": |- + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermission: "group_name": "description": |- diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index b08e0f4065..584f63ba8d 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -52,6 +52,13 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } + // Fields without a json tag (e.g. anonymous embeds) have no independent + // JSON key and cannot be required. Continue walking their children. + rawTag := field.Tag.Get("json") + if rawTag == "" { + return true + } + // Do not generate required validation code for fields that are internal or readonly. bundleTag := structtag.BundleTag(field.Tag.Get("bundle")) if bundleTag.Internal() || bundleTag.ReadOnly() { @@ -59,7 +66,7 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { } // The "omitempty" tag indicates the field is optional in bundle config. - jsonTag := structtag.JSONTag(field.Tag.Get("json")) + jsonTag := structtag.JSONTag(rawTag) if jsonTag.OmitEmpty() { return true } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 6e43f42291..0af9394f24 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,6 +25,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), + mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) } @@ -46,7 +47,6 @@ func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) - // If there is no prevent_destroy, skip preventDestroyV, err := dyn.GetByPath(root, path) if err != nil { continue diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 28b19d0e1e..73249b4e13 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -1,50 +1,32 @@ package phases import ( - "context" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/apps" - "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) func TestCheckPreventDestroyForAllResources(t *testing.T) { - supportedResources := config.SupportedResources() - - for resourceType := range supportedResources { + for resourceType := range config.SupportedResources() { t.Run(resourceType, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", - }, - Resources: config.Resources{}, - }, - } + b := &bundle.Bundle{} - ctx := t.Context() - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - // Use Mutate to set the configuration dynamically - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - // Set the resource with lifecycle.prevent_destroy = true - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - resourceType: dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + resourceType: dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), }, nil), }, nil), - }, nil)) - }) - require.NoError(t, err) + }, nil), + }, nil)) }) + require.NoError(t, err) actions := []deployplan.Action{ { @@ -53,7 +35,7 @@ func TestCheckPreventDestroyForAllResources(t *testing.T) { }, } - err := checkForPreventDestroy(b, actions) + err = checkForPreventDestroy(b, actions) require.Error(t, err) require.Contains(t, err.Error(), "resources."+resourceType+".test_resource has lifecycle.prevent_destroy set") require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") @@ -62,36 +44,102 @@ func TestCheckPreventDestroyForAllResources(t *testing.T) { } } -func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "test_job": { - JobSettings: jobs.JobSettings{ - Tasks: []jobs.Task{}, - }, - }, - }, - Apps: map[string]*resources.App{ - "test_app": { - App: apps.App{ - Name: "Test App", - }, - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - PreventDestroy: true, - }, - }, - }, - }, - }, +func TestCheckPreventDestroyForJob(t *testing.T) { + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "jobs": dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) + + actions := []deployplan.Action{ + { + ResourceKey: "resources.jobs.test_resource", + ActionType: deployplan.Recreate, + }, + } + + err = checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.jobs.test_resource has lifecycle.prevent_destroy set") + require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") + require.Contains(t, err.Error(), "disable lifecycle.prevent_destroy for resources.jobs.test_resource") +} + +func TestCheckPreventDestroyForApp(t *testing.T) { + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "apps": dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) + + actions := []deployplan.Action{ + { + ResourceKey: "resources.apps.test_resource", + ActionType: deployplan.Delete, + }, + } + + err = checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.apps.test_resource has lifecycle.prevent_destroy set") +} + +func TestCheckPreventDestroyNoError(t *testing.T) { + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "jobs": dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{}, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) + + actions := []deployplan.Action{ + { + ResourceKey: "resources.jobs.test_resource", + ActionType: deployplan.Recreate, }, } + err = checkForPreventDestroy(b, actions) + require.NoError(t, err) +} + +func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "jobs": dyn.NewValue(map[string]dyn.Value{ + "test_job": dyn.NewValue(map[string]dyn.Value{}, nil), + }, nil), + "apps": dyn.NewValue(map[string]dyn.Value{ + "test_app": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) + actions := []deployplan.Action{ { ResourceKey: "resources.jobs.test_job", @@ -103,10 +151,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { }, } - ctx := t.Context() - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := checkForPreventDestroy(b, actions) - require.Error(t, err) - require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") - }) + err = checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") } diff --git a/bundle/run/app.go b/bundle/run/app.go index 1dd4484093..c3a6497f1d 100644 --- a/bundle/run/app.go +++ b/bundle/run/app.go @@ -7,10 +7,10 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -124,7 +124,7 @@ func (a *appRunner) start(ctx context.Context) error { // active and pending deployments fields (if any). If there are active or pending deployments, // we need to wait for them to complete before we can do the new deployment. // Otherwise, the new deployment will fail. - err = waitForDeploymentToComplete(ctx, w, startedApp) + err = appdeploy.WaitForDeploymentToComplete(ctx, w, startedApp) if err != nil { return err } @@ -133,108 +133,10 @@ func (a *appRunner) start(ctx context.Context) error { return nil } -func waitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *apps.App) error { - // We first wait for the active deployment to complete. - if app.ActiveDeployment != nil && - app.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the active deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Active deployment is completed!") - } - - // Then, we wait for the pending deployment to complete. - if app.PendingDeployment != nil && - app.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the pending deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Pending deployment is completed!") - } - - return nil -} - -// buildAppDeployment creates an AppDeployment struct with inline config if provided -func (a *appRunner) buildAppDeployment() apps.AppDeployment { - deployment := apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: a.app.SourceCodePath, - } - - // Add git source if provided - if a.app.GitSource != nil { - deployment.GitSource = a.app.GitSource - } - - // Add inline config if provided - if a.app.Config != nil { - if len(a.app.Config.Command) > 0 { - deployment.Command = a.app.Config.Command - } - - if len(a.app.Config.Env) > 0 { - deployment.EnvVars = make([]apps.EnvVar, len(a.app.Config.Env)) - for i, env := range a.app.Config.Env { - deployment.EnvVars[i] = apps.EnvVar{ - Name: env.Name, - Value: env.Value, - ValueFrom: env.ValueFrom, - } - } - } - } - - return deployment -} - func (a *appRunner) deploy(ctx context.Context) error { - app := a.app - b := a.bundle - w := b.WorkspaceClient() - - wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - // If deploy returns an error, then there's an active deployment in progress, wait for it to complete. - // For this we first need to get an app and its acrive and pending deployments and then wait for them. - if err != nil { - app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name}) - if err != nil { - return fmt.Errorf("failed to get app %s: %w", app.Name, err) - } - - err = waitForDeploymentToComplete(ctx, w, app) - if err != nil { - return err - } - - // Now we can try to deploy the app again - wait, err = w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - if err != nil { - return err - } - } - - _, err = wait.OnProgress(func(ad *apps.AppDeployment) { - if ad.Status == nil { - return - } - logProgress(ctx, ad.Status.Message) - }).Get() - if err != nil { - return err - } - - return nil + w := a.bundle.WorkspaceClient() + deployment := appdeploy.BuildDeployment(a.app.SourceCodePath, a.app.Config, a.app.GitSource) + return appdeploy.Deploy(ctx, w, a.app.Name, deployment) } func (a *appRunner) Cancel(ctx context.Context) error { diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 112fcea28d..1c05fa63eb 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" @@ -294,11 +295,7 @@ func TestBuildAppDeploymentWithValueFrom(t *testing.T) { }, } - runner := &appRunner{ - app: app, - } - - deployment := runner.buildAppDeployment() + deployment := appdeploy.BuildDeployment(app.SourceCodePath, app.Config, app.GitSource) require.Equal(t, apps.AppDeploymentModeSnapshot, deployment.Mode) require.Equal(t, "/path/to/app", deployment.SourceCodePath) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index da37fcd786..993adec793 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -166,7 +166,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "name": { "description": "The name of the app. The name must contain only lowercase alphanumeric characters and hyphens.\nIt must be unique within the workspace.", @@ -905,6 +905,28 @@ } ] }, + "resources.LifecycleWithStarted": { + "oneOf": [ + { + "type": "object", + "properties": { + "prevent_destroy": { + "description": "Lifecycle setting to prevent the resource from being destroyed.", + "$ref": "#/$defs/bool" + }, + "started": { + "description": "Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.MlflowExperiment": { "oneOf": [ { diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 8abd4cd416..3a4521266b 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -89,6 +89,83 @@ func (s *FakeWorkspace) AppsGetUpdate(_ Request, name string) Response { } } +func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + var deployment apps.AppDeployment + if err := json.Unmarshal(req.Body, &deployment); err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + + deployment.DeploymentId = fmt.Sprintf("deploy-%d", nextID()) + deployment.Status = &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + } + + app.ActiveDeployment = &deployment + app.DefaultSourceCodePath = deployment.SourceCodePath + s.Apps[name] = app + + return Response{Body: deployment} +} + +func (s *FakeWorkspace) AppsGetDeployment(_ Request, name, deploymentID string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: apps.AppDeployment{ + DeploymentId: deploymentID, + Status: &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + }, + }} +} + +func (s *FakeWorkspace) AppsStart(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateActive, + Message: "App compute is active.", + } + s.Apps[name] = app + + return Response{Body: app} +} + +func (s *FakeWorkspace) AppsStop(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateStopped, + Message: "App compute is stopped.", + } + s.Apps[name] = app + + return Response{Body: app} +} + func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { var app apps.App @@ -133,9 +210,17 @@ func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { Message: "Application is running.", } - app.ComputeStatus = &apps.ComputeStatus{ - State: "ACTIVE", - Message: "App compute is active.", + // Respect no_compute query param: if true, start the app in STOPPED state. + if req.URL.Query().Get("no_compute") == "true" { + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateStopped, + Message: "App compute is stopped.", + } + } else { + app.ComputeStatus = &apps.ComputeStatus{ + State: "ACTIVE", + Message: "App compute is active.", + } } app.Url = name + "-123.cloud.databricksapps.com" diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9e30cb5f0c..b2a95b1902 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -379,6 +379,22 @@ func AddDefaultHandlers(server *Server) { // Apps: + server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req Request) any { + return req.Workspace.AppsCreateDeployment(req, req.Vars["name"]) + }) + + server.Handle("GET", "/api/2.0/apps/{name}/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.AppsGetDeployment(req, req.Vars["name"], req.Vars["deployment_id"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req Request) any { + return req.Workspace.AppsStart(req, req.Vars["name"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/stop", func(req Request) any { + return req.Workspace.AppsStop(req, req.Vars["name"]) + }) + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req Request) any { return req.Workspace.AppsCreateUpdate(req, req.Vars["name"]) })