Skip to content

Commit 1241eb8

Browse files
denikdenik
authored
direct: support quality monitors (#4278)
## Changes Support quality monitors in direct engine. Compared to terraform, this supports changing assets_dir and table_name (by recreating the monitor). ## Why Closes #4158 ## Tests New local + cloud tests.
1 parent c75194b commit 1241eb8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1046
-3
lines changed

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ To disable this, set the environment variable DATABRICKS_CACHE_ENABLED to false.
1616
* Fix false positive folder permission warnings and make them more actionable ([#4216](https://github.com/databricks/cli/pull/4216))
1717
* Pass additional Azure DevOps system variables ([#4236](https://github.com/databricks/cli/pull/4236))
1818
* Replace Black formatter with Ruff in Python bundle templates for faster, all-in-one linting and formatting ([#4196](https://github.com/databricks/cli/pull/4196))
19+
* engine/direct: support quality monitors ([#4278](https://github.com/databricks/cli/pull/4278))
1920

2021
### Dependency updates
2122

‎acceptance/bin/create_table.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/usr/bin/env python3
2+
import json
3+
import os
4+
import subprocess
5+
import sys
6+
import time
7+
8+
table_name = sys.argv[1]
9+
# Extract catalog.schema from table_name
10+
parts = table_name.split(".")
11+
if len(parts) != 3:
12+
print(f"Invalid table name: {table_name}. Expected format: catalog.schema.table", file=sys.stderr)
13+
sys.exit(1)
14+
15+
catalog_name = parts[0]
16+
schema_name = parts[1]
17+
full_schema_name = f"{catalog_name}.{schema_name}"
18+
19+
cli = os.environ.get("CLI", "databricks")
20+
21+
22+
def run_cli(*args):
23+
result = subprocess.run([cli, *args], capture_output=True, text=True)
24+
return result
25+
26+
27+
def execute_sql(warehouse_id, sql):
28+
"""Execute SQL using the API endpoint."""
29+
payload = json.dumps({"warehouse_id": warehouse_id, "statement": sql, "wait_timeout": "30s"})
30+
return run_cli("api", "post", "/api/2.0/sql/statements/", "--json", payload)
31+
32+
33+
# Get warehouse ID from environment variable
34+
warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"]
35+
36+
# Create a simple table
37+
sql = f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA"
38+
39+
result = execute_sql(warehouse_id, sql)
40+
if result.returncode != 0:
41+
print(f"Failed to create table: {result.stderr}", file=sys.stderr)
42+
sys.exit(1)
43+
44+
print(f"Created table {table_name}")
45+
46+
# Insert some sample data so the monitor has something to analyze
47+
insert_sql = f"""INSERT INTO {table_name} VALUES
48+
(1, 'test1', current_timestamp()),
49+
(2, 'test2', current_timestamp()),
50+
(3, 'test3', current_timestamp())"""
51+
52+
result = execute_sql(warehouse_id, insert_sql)
53+
if result.returncode != 0:
54+
print(f"Failed to insert data: {result.stderr}", file=sys.stderr)
55+
sys.exit(1)
56+
57+
print(f"Inserted sample data into {table_name}")
58+
59+
# Wait for table to be visible in Unity Catalog
60+
for attempt in range(10):
61+
result = run_cli("tables", "get", table_name)
62+
if result.returncode == 0:
63+
table_info = json.loads(result.stdout)
64+
print(f"Table {table_name} is now visible (catalog_name={table_info.get('catalog_name')})")
65+
break
66+
if attempt < 9:
67+
time.sleep(1)
68+
else:
69+
print(f"Warning: Table may not be immediately visible: {result.stderr}", file=sys.stderr)

‎acceptance/bundle/deployment/bind/quality-monitor/output.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
>>> [CLI] quality-monitors create catalog.schema.table --json @input.json
33
{
44
"assets_dir":"/Users/user/databricks_lakehouse_monitoring",
5-
"drift_metrics_table_name":"",
5+
"dashboard_id":"(redacted)",
6+
"drift_metrics_table_name":"catalog.schema.table_drift_metrics",
67
"monitor_version":0,
78
"output_schema_name":"catalog.schema",
8-
"profile_metrics_table_name":"",
9+
"profile_metrics_table_name":"catalog.schema.table_profile_metrics",
910
"snapshot": {},
1011
"status":"MONITOR_STATUS_ACTIVE",
1112
"table_name":"catalog.schema.table"

‎acceptance/bundle/deployment/bind/quality-monitor/test.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
Local = true
22
Cloud = false
3+
4+
[[Repls]]
5+
Old = '"dashboard_id":"[0-9a-f]+",'
6+
New = '"dashboard_id":"(redacted)",'

‎acceptance/bundle/refschema/out.fields.txt‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3464,6 +3464,60 @@ resources.pipelines.*.permissions.permissions[*].group_name string ALL
34643464
resources.pipelines.*.permissions.permissions[*].permission_level iam.PermissionLevel ALL
34653465
resources.pipelines.*.permissions.permissions[*].service_principal_name string ALL
34663466
resources.pipelines.*.permissions.permissions[*].user_name string ALL
3467+
resources.quality_monitors.*.assets_dir string ALL
3468+
resources.quality_monitors.*.baseline_table_name string ALL
3469+
resources.quality_monitors.*.custom_metrics []catalog.MonitorMetric ALL
3470+
resources.quality_monitors.*.custom_metrics[*] catalog.MonitorMetric ALL
3471+
resources.quality_monitors.*.custom_metrics[*].definition string ALL
3472+
resources.quality_monitors.*.custom_metrics[*].input_columns []string ALL
3473+
resources.quality_monitors.*.custom_metrics[*].input_columns[*] string ALL
3474+
resources.quality_monitors.*.custom_metrics[*].name string ALL
3475+
resources.quality_monitors.*.custom_metrics[*].output_data_type string ALL
3476+
resources.quality_monitors.*.custom_metrics[*].type catalog.MonitorMetricType ALL
3477+
resources.quality_monitors.*.dashboard_id string REMOTE
3478+
resources.quality_monitors.*.data_classification_config *catalog.MonitorDataClassificationConfig ALL
3479+
resources.quality_monitors.*.data_classification_config.enabled bool ALL
3480+
resources.quality_monitors.*.drift_metrics_table_name string REMOTE
3481+
resources.quality_monitors.*.id string INPUT
3482+
resources.quality_monitors.*.inference_log *catalog.MonitorInferenceLog ALL
3483+
resources.quality_monitors.*.inference_log.granularities []string ALL
3484+
resources.quality_monitors.*.inference_log.granularities[*] string ALL
3485+
resources.quality_monitors.*.inference_log.label_col string ALL
3486+
resources.quality_monitors.*.inference_log.model_id_col string ALL
3487+
resources.quality_monitors.*.inference_log.prediction_col string ALL
3488+
resources.quality_monitors.*.inference_log.prediction_proba_col string ALL
3489+
resources.quality_monitors.*.inference_log.problem_type catalog.MonitorInferenceLogProblemType ALL
3490+
resources.quality_monitors.*.inference_log.timestamp_col string ALL
3491+
resources.quality_monitors.*.latest_monitor_failure_msg string ALL
3492+
resources.quality_monitors.*.lifecycle resources.Lifecycle INPUT
3493+
resources.quality_monitors.*.lifecycle.prevent_destroy bool INPUT
3494+
resources.quality_monitors.*.modified_status string INPUT
3495+
resources.quality_monitors.*.monitor_version int64 REMOTE
3496+
resources.quality_monitors.*.notifications *catalog.MonitorNotifications ALL
3497+
resources.quality_monitors.*.notifications.on_failure *catalog.MonitorDestination ALL
3498+
resources.quality_monitors.*.notifications.on_failure.email_addresses []string ALL
3499+
resources.quality_monitors.*.notifications.on_failure.email_addresses[*] string ALL
3500+
resources.quality_monitors.*.notifications.on_new_classification_tag_detected *catalog.MonitorDestination ALL
3501+
resources.quality_monitors.*.notifications.on_new_classification_tag_detected.email_addresses []string ALL
3502+
resources.quality_monitors.*.notifications.on_new_classification_tag_detected.email_addresses[*] string ALL
3503+
resources.quality_monitors.*.output_schema_name string ALL
3504+
resources.quality_monitors.*.profile_metrics_table_name string REMOTE
3505+
resources.quality_monitors.*.schedule *catalog.MonitorCronSchedule ALL
3506+
resources.quality_monitors.*.schedule.pause_status catalog.MonitorCronSchedulePauseStatus ALL
3507+
resources.quality_monitors.*.schedule.quartz_cron_expression string ALL
3508+
resources.quality_monitors.*.schedule.timezone_id string ALL
3509+
resources.quality_monitors.*.skip_builtin_dashboard bool INPUT STATE
3510+
resources.quality_monitors.*.slicing_exprs []string ALL
3511+
resources.quality_monitors.*.slicing_exprs[*] string ALL
3512+
resources.quality_monitors.*.snapshot *catalog.MonitorSnapshot ALL
3513+
resources.quality_monitors.*.status catalog.MonitorInfoStatus REMOTE
3514+
resources.quality_monitors.*.table_name string ALL
3515+
resources.quality_monitors.*.time_series *catalog.MonitorTimeSeries ALL
3516+
resources.quality_monitors.*.time_series.granularities []string ALL
3517+
resources.quality_monitors.*.time_series.granularities[*] string ALL
3518+
resources.quality_monitors.*.time_series.timestamp_col string ALL
3519+
resources.quality_monitors.*.url string INPUT
3520+
resources.quality_monitors.*.warehouse_id string INPUT STATE
34673521
resources.registered_models.*.aliases []catalog.RegisteredModelAlias ALL
34683522
resources.registered_models.*.aliases[*] catalog.RegisteredModelAlias ALL
34693523
resources.registered_models.*.aliases[*].alias_name string ALL

‎acceptance/bundle/resources/quality_monitors/change_assets_dir/databricks.yml.tmpl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
bundle:
2+
name: quality-monitor-update-$UNIQUE_NAME
3+
4+
resources:
5+
quality_monitors:
6+
monitor1:
7+
table_name: main.qm_test_$UNIQUE_NAME.test_table
8+
assets_dir: /Workspace/Users/$CURRENT_USER_NAME/monitor_assets_$UNIQUE_NAME
9+
output_schema_name: main.qm_test_$UNIQUE_NAME
10+
snapshot: {}
11+
warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID

‎acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.direct.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
>>> errcode [CLI] bundle deploy
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!

‎acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.direct.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"method": "DELETE",
3+
"path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor"
4+
}
5+
{
6+
"method": "POST",
7+
"path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor",
8+
"body": {
9+
"assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets2_[UNIQUE_NAME]",
10+
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
11+
"snapshot": {},
12+
"warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
13+
}
14+
}

‎acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.requests.terraform.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"method": "PUT",
3+
"path": "/api/2.1/unity-catalog/tables/main.qm_test_[UNIQUE_NAME].test_table/monitor",
4+
"body": {
5+
"dashboard_id": "(redacted)",
6+
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
7+
"snapshot": {}
8+
}
9+
}

‎acceptance/bundle/resources/quality_monitors/change_assets_dir/out.deploy.terraform.txt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
2+
>>> errcode [CLI] bundle deploy
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files...
4+
Deploying resources...
5+
Error: terraform apply: exit status 1
6+
7+
Error: Provider produced inconsistent result after apply
8+
9+
When applying changes to databricks_quality_monitor.monitor1, provider
10+
"provider[\"registry.terraform.io/databricks/databricks\"]" produced an
11+
unexpected new value: .assets_dir: was
12+
cty.StringVal("/Workspace/Users/[USERNAME]/monitor_assets2_[UNIQUE_NAME]"),
13+
but now
14+
cty.StringVal("/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]").
15+
16+
This is a bug in the provider, which should be reported in the provider's own
17+
issue tracker.
18+
19+
20+
Updating deployment state...
21+
22+
Exit code: 1

0 commit comments

Comments
 (0)