diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 3e67081803..ac1c24d104 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -2cee201b2e8d656f7306b2f9ec98edfa721e9829 \ No newline at end of file +a8f547d3728fba835fbdda301e846829c5cbbef5 \ No newline at end of file diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl index b28e565c19..c8d50adfaf 100644 --- a/.codegen/service.go.tmpl +++ b/.codegen/service.go.tmpl @@ -203,9 +203,9 @@ func new{{.PascalName}}() *cobra.Command { {{- end -}} cmd.Use = "{{.KebabName}}{{if $hasPosArgs}}{{range .RequiredPositionalArguments}} {{.ConstantName}}{{end}}{{end}}" - {{- if .Description }} + {{- if .CliComment " " 80 }} cmd.Short = `{{.Summary | without "`"}}` - cmd.Long = `{{.Comment " " 80 | without "`"}} + cmd.Long = `{{.CliComment " " 80 | without "`"}} {{- if $atleastOneArgumentWithDescription }} Arguments: @@ -412,7 +412,7 @@ func new{{.PascalName}}() *cobra.Command { {{- if $optionalIfJsonIsUsed }} if !cmd.Flags().Changed("json") { {{- end }} - {{if and (not $field.Entity.IsString) (not $field.Entity.IsFieldMask) (not $field.Entity.IsTimestamp) (not $field.Entity.IsDuration) -}} {{/* TODO: add support for well known types */}} + {{if and (not $field.Entity.IsString) (not $field.Entity.IsFieldMask) (not $field.Entity.IsTimestamp) (not $field.Entity.IsDuration) -}} {{/* TODO: add support for well known types */ -}} _, err = fmt.Sscan(args[{{$arg}}], &{{- template "request-body-obj" (dict "Method" $method "Field" $field)}}) if err != nil { return fmt.Errorf("invalid {{$field.ConstantName}}: %s", args[{{$arg}}]) diff --git a/.databricks/databricks b/.databricks/databricks new file mode 100755 index 0000000000..e4218e3d03 Binary files /dev/null and b/.databricks/databricks differ diff --git a/.databricks/dlt b/.databricks/dlt new file mode 100755 index 0000000000..e4218e3d03 Binary files /dev/null and b/.databricks/dlt differ diff --git a/.gitattributes b/.gitattributes index 629be14230..66a1ee60c5 100755 --- a/.gitattributes +++ b/.gitattributes @@ -63,10 +63,12 @@ cmd/workspace/consumer-providers/consumer-providers.go linguist-generated=true cmd/workspace/credentials-manager/credentials-manager.go linguist-generated=true cmd/workspace/credentials/credentials.go linguist-generated=true cmd/workspace/current-user/current-user.go linguist-generated=true +cmd/workspace/custom-llms/custom-llms.go linguist-generated=true +cmd/workspace/dashboard-email-subscriptions/dashboard-email-subscriptions.go linguist-generated=true cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true cmd/workspace/dashboards/dashboards.go linguist-generated=true cmd/workspace/data-sources/data-sources.go linguist-generated=true -cmd/workspace/database-instances/database-instances.go linguist-generated=true +cmd/workspace/database/database.go linguist-generated=true cmd/workspace/default-namespace/default-namespace.go linguist-generated=true cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true cmd/workspace/disable-legacy-dbfs/disable-legacy-dbfs.go linguist-generated=true @@ -110,10 +112,10 @@ cmd/workspace/provider-personalization-requests/provider-personalization-request cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go linguist-generated=true cmd/workspace/provider-providers/provider-providers.go linguist-generated=true cmd/workspace/providers/providers.go linguist-generated=true +cmd/workspace/quality-monitor-v2/quality-monitor-v2.go linguist-generated=true cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true cmd/workspace/queries-legacy/queries-legacy.go linguist-generated=true cmd/workspace/queries/queries.go linguist-generated=true -cmd/workspace/query-execution/query-execution.go linguist-generated=true cmd/workspace/query-history/query-history.go linguist-generated=true cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go linguist-generated=true cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true @@ -131,6 +133,7 @@ cmd/workspace/service-principals/service-principals.go linguist-generated=true cmd/workspace/serving-endpoints/serving-endpoints.go linguist-generated=true cmd/workspace/settings/settings.go linguist-generated=true cmd/workspace/shares/shares.go linguist-generated=true +cmd/workspace/sql-results-download/sql-results-download.go linguist-generated=true cmd/workspace/storage-credentials/storage-credentials.go linguist-generated=true cmd/workspace/system-schemas/system-schemas.go linguist-generated=true cmd/workspace/table-constraints/table-constraints.go linguist-generated=true diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml index 108ca91626..3c3e517eb1 100644 --- a/.github/workflows/external-message.yml +++ b/.github/workflows/external-message.yml @@ -29,25 +29,25 @@ jobs: - name: Delete old comments env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # Delete previous comment if it exists - previous_comment_ids=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ - --jq '.[] | select(.body | startswith("")) | .id') - echo "Previous comment IDs: $previous_comment_ids" - # Iterate over each comment ID and delete the comment - if [ ! -z "$previous_comment_ids" ]; then - echo "$previous_comment_ids" | while read -r comment_id; do - echo "Deleting comment with ID: $comment_id" - gh api "repos/${{ github.repository }}/issues/comments/$comment_id" -X DELETE - done - fi + # Delete previous comment if it exists + previous_comment_ids=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ + --jq '.[] | select(.body | startswith("")) | .id') + echo "Previous comment IDs: $previous_comment_ids" + # Iterate over each comment ID and delete the comment + if [ ! -z "$previous_comment_ids" ]; then + echo "$previous_comment_ids" | while read -r comment_id; do + echo "Deleting comment with ID: $comment_id" + gh api "repos/${{ github.repository }}/issues/comments/$comment_id" -X DELETE + done + fi - name: Comment on PR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMMIT_SHA: ${{ github.event.pull_request.head.sha }} - run: | + run: |- gh pr comment ${{ github.event.pull_request.number }} --body \ " An authorized user can trigger integration tests manually by following the instructions below: diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml index 293d31a2a4..7598bf453d 100644 --- a/.github/workflows/integration-approve.yml +++ b/.github/workflows/integration-approve.yml @@ -26,7 +26,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash - run: | + run: |- gh api -X POST -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ /repos/${{ github.repository }}/statuses/${{ github.sha }} \ diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml index ab8497497e..96e13be29a 100644 --- a/.github/workflows/integration-main.yml +++ b/.github/workflows/integration-main.yml @@ -30,7 +30,7 @@ jobs: - name: Trigger Workflow in Another Repo env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} - run: | + run: |- gh workflow run cli-isolated-nightly.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \ --ref main \ -f commit_sha=${{ github.event.after }} diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml index 49b4ca0bf8..4ebb242564 100644 --- a/.github/workflows/integration-pr.yml +++ b/.github/workflows/integration-pr.yml @@ -33,7 +33,7 @@ jobs: - name: Trigger Workflow in Another Repo env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} - run: | + run: |- gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \ --ref main \ -f pull_request_number=${{ github.event.pull_request.number }} \ diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index da0408a505..945c7367e6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -14,7 +14,7 @@ on: branches: - main schedule: - - cron: '0 0,12 * * *' # Runs at 00:00 and 12:00 UTC daily + - cron: '0 0,12 * * *' # Runs at 00:00 and 12:00 UTC daily env: GOTESTSUM_FORMAT: github-actions @@ -112,10 +112,10 @@ jobs: with: version: "0.9.1" args: "format --check" - - name: "make fmt: Python and Go formatting" + - name: "make fmtfull: Python and Go formatting" # This is already done by actions, but good to check that make command is working run: | - make fmt + make fmtfull git diff --exit-code - name: "make checks: custom checks outside of fmt and lint" run: | @@ -192,7 +192,7 @@ jobs: - name: Verify that python/codegen is up to date working-directory: experimental/python - run: | + run: |- make codegen if ! ( git diff --exit-code ); then diff --git a/.github/workflows/python_push.yml b/.github/workflows/python_push.yml index 5405d11642..a4989f31d6 100644 --- a/.github/workflows/python_push.yml +++ b/.github/workflows/python_push.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - pyVersion: [ '3.10', '3.11', '3.12', '3.13' ] + pyVersion: ['3.10', '3.11', '3.12', '3.13'] steps: - name: Checkout repository and submodules diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml index 2f99b4ac8f..da3d5e247f 100644 --- a/.github/workflows/release-snapshot.yml +++ b/.github/workflows/release-snapshot.yml @@ -94,6 +94,6 @@ jobs: prerelease: true tag_name: snapshot token: ${{ secrets.GITHUB_TOKEN }} - files: | + files: |- dist/databricks_cli_*.zip dist/databricks_cli_*.tar.gz diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fde40f646..effe1dba57 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,7 +161,6 @@ jobs: needs: goreleaser - # IMPORTANT: # - 'id-token: write' is mandatory for OIDC and trusted publishing to PyPi # - 'environment: release' is a part of OIDC assertion done by PyPi diff --git a/.github/workflows/start-integration-tests.yml b/.github/workflows/start-integration-tests.yml index 72392ab6d2..66ecfde2d4 100644 --- a/.github/workflows/start-integration-tests.yml +++ b/.github/workflows/start-integration-tests.yml @@ -37,5 +37,5 @@ jobs: - name: Run start_integration_tests.py env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} - run: | + run: |- python3 ./start_integration_tests.py -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} --yes diff --git a/.github/workflows/tagging.yml b/.github/workflows/tagging.yml index b6b816da06..15c8060dd9 100644 --- a/.github/workflows/tagging.yml +++ b/.github/workflows/tagging.yml @@ -11,7 +11,6 @@ on: concurrency: group: "tagging" - jobs: tag: environment: "release-is" @@ -47,5 +46,5 @@ jobs: env: GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} - run: | + run: |- python internal/genkit/tagging.py diff --git a/.gitignore b/.gitignore index 403d6606df..d9cdc190a9 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ __pycache__ # Test results from 'make test' test-output.json + +# Built by make for 'make fmt' and yamlcheck.py in acceptance tests +tools/yamlfmt +tools/yamlfmt.exe diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d2f4f318cc..8471d8410f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,40 +5,40 @@ before: - go mod download builds: -- env: - - CGO_ENABLED=0 - mod_timestamp: '{{ .CommitTimestamp }}' - flags: - - -trimpath - ldflags: - - '-s -w' - - -X github.com/databricks/cli/internal/build.buildProjectName={{ .ProjectName }} - - -X github.com/databricks/cli/internal/build.buildVersion={{ .Version }} + - env: + - CGO_ENABLED=0 + mod_timestamp: '{{ .CommitTimestamp }}' + flags: + - -trimpath + ldflags: + - '-s -w' + - -X github.com/databricks/cli/internal/build.buildProjectName={{ .ProjectName }} + - -X github.com/databricks/cli/internal/build.buildVersion={{ .Version }} - # Git information - - -X github.com/databricks/cli/internal/build.buildBranch={{ .Branch }} - - -X github.com/databricks/cli/internal/build.buildTag={{ .Tag }} - - -X github.com/databricks/cli/internal/build.buildShortCommit={{ .ShortCommit }} - - -X github.com/databricks/cli/internal/build.buildFullCommit={{ .FullCommit }} - - -X github.com/databricks/cli/internal/build.buildCommitTimestamp={{ .CommitTimestamp }} - - -X github.com/databricks/cli/internal/build.buildSummary={{ .Summary }} + # Git information + - -X github.com/databricks/cli/internal/build.buildBranch={{ .Branch }} + - -X github.com/databricks/cli/internal/build.buildTag={{ .Tag }} + - -X github.com/databricks/cli/internal/build.buildShortCommit={{ .ShortCommit }} + - -X github.com/databricks/cli/internal/build.buildFullCommit={{ .FullCommit }} + - -X github.com/databricks/cli/internal/build.buildCommitTimestamp={{ .CommitTimestamp }} + - -X github.com/databricks/cli/internal/build.buildSummary={{ .Summary }} - # Version information - - -X github.com/databricks/cli/internal/build.buildMajor={{ .Major }} - - -X github.com/databricks/cli/internal/build.buildMinor={{ .Minor }} - - -X github.com/databricks/cli/internal/build.buildPatch={{ .Patch }} - - -X github.com/databricks/cli/internal/build.buildPrerelease={{ .Prerelease }} - - -X github.com/databricks/cli/internal/build.buildIsSnapshot={{ .IsSnapshot }} - - -X github.com/databricks/cli/internal/build.buildTimestamp={{ .Timestamp }} + # Version information + - -X github.com/databricks/cli/internal/build.buildMajor={{ .Major }} + - -X github.com/databricks/cli/internal/build.buildMinor={{ .Minor }} + - -X github.com/databricks/cli/internal/build.buildPatch={{ .Patch }} + - -X github.com/databricks/cli/internal/build.buildPrerelease={{ .Prerelease }} + - -X github.com/databricks/cli/internal/build.buildIsSnapshot={{ .IsSnapshot }} + - -X github.com/databricks/cli/internal/build.buildTimestamp={{ .Timestamp }} - goos: - - windows - - linux - - darwin - goarch: - - amd64 - - arm64 - binary: databricks + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - arm64 + binary: databricks archives: - formats: ["zip", "tar.gz"] @@ -89,7 +89,6 @@ docker_manifests: - ghcr.io/databricks/cli:latest-amd64 - ghcr.io/databricks/cli:latest-arm64 - checksum: name_template: 'databricks_cli_{{ .Version }}_SHA256SUMS' algorithm: sha256 @@ -101,5 +100,5 @@ changelog: sort: asc filters: exclude: - - '^docs:' - - '^test:' + - '^docs:' + - '^test:' diff --git a/.release_metadata.json b/.release_metadata.json index 8ae1d72e67..aa322452fb 100644 --- a/.release_metadata.json +++ b/.release_metadata.json @@ -1,3 +1,3 @@ { - "timestamp": "2025-06-03 13:30:49+0000" + "timestamp": "2025-06-11 14:09:54+0000" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index bad5795f57..e1eacd6a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Version changelog +## Release v0.255.0 + +### Notable Changes + +* Fix `databricks auth login` to tolerate URLs copied from the browser ([#3001](https://github.com/databricks/cli/pull/3001)). + +### CLI +* Use OS aware runner instead of bash for run-local command ([#2996](https://github.com/databricks/cli/pull/2996)) + +### Bundles +* Fix "bundle summary -o json" to render null values properly ([#2990](https://github.com/databricks/cli/pull/2990)) +* Fix dashboard generation for already imported dashboard ([#3016](https://github.com/databricks/cli/pull/3016)) +* Fixed null pointer de-reference if artifacts missing fields ([#3022](https://github.com/databricks/cli/pull/3022)) +* Update bundle templates to also include `resources/*/*.yml` ([#3024](https://github.com/databricks/cli/pull/3024)) +* Apply YAML formatter on default-python and dbt-sql templates ([#3026](https://github.com/databricks/cli/pull/3026)) + + ## Release v0.254.0 ### Bundles diff --git a/Makefile b/Makefile index 776a784e9a..0ea45eba93 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,12 @@ GOTESTSUM_FORMAT ?= pkgname-and-test-fails GOTESTSUM_CMD ?= go tool gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped --jsonfile test-output.json -lint: +lintfull: golangci-lint run --fix +lint: + ./tools/lintdiff.py run --fix + tidy: @# not part of golangci-lint, apparently go mod tidy @@ -16,9 +19,22 @@ tidy: lintcheck: golangci-lint run ./... -fmt: - ruff format -qn +fmtfull: tools/yamlfmt + ruff format -n golangci-lint fmt + ./tools/yamlfmt . + +fmt: tools/yamlfmt + ruff format -n + ./tools/lintdiff.py fmt + ./tools/yamlfmt . + +# pre-building yamlfmt because I also want to call it from tests +tools/yamlfmt: go.mod + go build -o tools/yamlfmt github.com/google/yamlfmt/cmd/yamlfmt + +tools/yamlfmt.exe: go.mod + go build -o tools/yamlfmt.exe github.com/google/yamlfmt/cmd/yamlfmt ws: ./tools/validate_whitespace.py @@ -54,7 +70,6 @@ build: tidy snapshot: go build -o .databricks/databricks - go build -o .databricks/dlt schema: go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json @@ -79,4 +94,4 @@ generate: [ ! -f .github/workflows/next-changelog.yml ] || rm .github/workflows/next-changelog.yml pushd experimental/python && make codegen -.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot schema integration integration-short acc-cover acc-showcover docs ws links checks +.PHONY: lint lintfull tidy lintcheck fmt fmtfull test cover showcover build snapshot schema integration integration-short acc-cover acc-showcover docs ws links checks diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index db0efc7d8f..c6e54dc1fe 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -1,15 +1,17 @@ # NEXT CHANGELOG -## Release v0.255.0 +## Release v0.256.0 ### Notable Changes ### Dependency updates ### CLI -* Use OS aware runner instead of bash for run-local command ([#2996](https://github.com/databricks/cli/pull/2996)) ### Bundles -* Fix "bundle summary -o json" to render null values properly ([#2990](https://github.com/databricks/cli/pull/2990)) +* Fix reading dashboard contents when the sync root is different than the bundle root ([#3006](https://github.com/databricks/cli/pull/3006)) +* When glob for wheels is used, like "\*.whl", it will filter out different version of the same package and will only take the most recent version. ([#2982](https://github.com/databricks/cli/pull/2982)) +* When building Python artifacts as part of "bundle deploy" we no longer delete `dist`, `build`, `*egg-info` and `__pycache__` directories. ([#2982](https://github.com/databricks/cli/pull/2982)) +* Fix variable resolution for lookup variables with other references ([#3054](https://github.com/databricks/cli/pull/3054)) ### API Changes diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 929d5b32c6..232cf48ab5 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -81,6 +81,13 @@ const ( EnvFilterVar = "ENVFILTER" ) +var exeSuffix = func() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" +}() + var Scripts = map[string]bool{ EntryPointScript: true, CleanupScript: true, @@ -137,21 +144,49 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { } execPath := "" + dltPath := "" if inprocessMode { cmdServer := internal.StartCmdServer(t) t.Setenv("CMD_SERVER_URL", cmdServer.URL) execPath = filepath.Join(cwd, "bin", "callserver.py") + dltPath = filepath.Join(buildDir, "dlt") } else { - execPath = BuildCLI(t, buildDir, coverDir) + execPath, dltPath = BuildCLI(t, buildDir, coverDir) + } + + fi, err := os.Lstat(dltPath) + if err == nil { + if fi.Mode()&os.ModeSymlink == 0 { + _ = os.Remove(dltPath) + } + } else if !os.IsNotExist(err) { + require.NoError(t, err) } + err = os.Symlink(execPath, dltPath) + if err != nil && !os.IsExist(err) { + require.NoError(t, err) + } + + BuildYamlfmt(t) t.Setenv("CLI", execPath) repls.SetPath(execPath, "[CLI]") - // Make helper scripts available - pathDirs := []string{filepath.Join(cwd, "bin"), buildDir} - t.Setenv("PATH", fmt.Sprintf("%s%c%s", strings.Join(pathDirs, string(os.PathListSeparator)), os.PathListSeparator, os.Getenv("PATH"))) + t.Setenv("DLT", dltPath) + repls.SetPath(dltPath, "[DLT]") + + t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH"))) + paths := []string{ + // Make helper scripts available + filepath.Join(cwd, "bin"), + + // Make /tools/ available (e.g. yamlfmt) + filepath.Join(cwd, "..", "tools"), + + os.Getenv("PATH"), + } + t.Setenv("PATH", strings.Join(paths, string(os.PathListSeparator))) tempHomeDir := t.TempDir() repls.SetPath(tempHomeDir, "[TMPHOME]") @@ -177,10 +212,7 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]") - terraformExecPath := filepath.Join(buildDir, "terraform") - if runtime.GOOS == "windows" { - terraformExecPath += ".exe" - } + terraformExecPath := filepath.Join(buildDir, "terraform") + exeSuffix t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) t.Setenv("TERRAFORM", terraformExecPath) repls.SetPath(terraformExecPath, "[TERRAFORM]") @@ -194,6 +226,7 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { testdiff.PrepareReplacementSdkVersion(t, &repls) testdiff.PrepareReplacementsGoVersion(t, &repls) + t.Setenv("TESTROOT", cwd) repls.SetPath(cwd, "[TESTROOT]") repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) @@ -723,10 +756,13 @@ func readMergedScriptContents(t *testing.T, dir string) string { return strings.Join(prepares, "\n") } -func BuildCLI(t *testing.T, buildDir, coverDir string) string { +func BuildCLI(t *testing.T, buildDir, coverDir string) (string, string) { execPath := filepath.Join(buildDir, "databricks") + dltPath := filepath.Join(buildDir, "dlt") + if runtime.GOOS == "windows" { execPath += ".exe" + dltPath += ".exe" } args := []string{ @@ -745,7 +781,7 @@ func BuildCLI(t *testing.T, buildDir, coverDir string) string { } RunCommand(t, args, "..") - return execPath + return execPath, dltPath } func copyFile(src, dst string) error { @@ -1138,3 +1174,11 @@ func isSameYAMLContent(str1, str2 string) bool { return reflect.DeepEqual(obj1, obj2) } + +func BuildYamlfmt(t *testing.T) { + // Using make here instead of "go build" directly cause it's faster when it's already built + args := []string{ + "make", "-s", "tools/yamlfmt" + exeSuffix, + } + RunCommand(t, args, "..") +} diff --git a/acceptance/bin/yamlcheck.py b/acceptance/bin/yamlcheck.py new file mode 100755 index 0000000000..eca62517e8 --- /dev/null +++ b/acceptance/bin/yamlcheck.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +from pathlib import Path +from difflib import unified_diff + + +NAME = "yamlfmt" +if sys.platform.startswith("win"): + NAME += ".exe" + + +def main(): + current_dir = Path.cwd() + yaml_files = sorted(current_dir.glob("**/*.yml")) + sorted(current_dir.glob("**/*.yaml")) + if not yaml_files: + sys.exit("No YAML files found") + + yamlfmt_conf = Path(os.environ["TESTROOT"]) / ".." / "yamlfmt.yml" + yamlfmt = Path(os.environ["TESTROOT"]) / ".." / "tools" / NAME + + has_changes = [] + + for yaml_file in yaml_files: + original_content = yaml_file.read_text().splitlines(keepends=True) + + subprocess.run([yamlfmt, f"-conf={yamlfmt_conf}", str(yaml_file)], check=True, capture_output=True) + + formatted_content = yaml_file.read_text().splitlines(keepends=True) + + if original_content != formatted_content: + has_changes.append(str(yaml_file)) + # Add $ markers for trailing whitespace + original_with_markers = [line.rstrip("\n") + ("$" if line.rstrip() != line.rstrip("\n") else "") + "\n" for line in original_content] + formatted_with_markers = [line.rstrip("\n") + ("$" if line.rstrip() != line.rstrip("\n") else "") + "\n" for line in formatted_content] + diff = unified_diff(original_with_markers, formatted_with_markers, fromfile=str(yaml_file), tofile=str(yaml_file), lineterm="") + print("".join(diff)) + + if has_changes: + sys.exit("UNEXPECTED: YAML formatting issues in " + " ".join(has_changes)) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml b/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml index 832f8b1d91..9f737e9558 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml +++ b/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml @@ -1,8 +1,8 @@ artifacts: - my_prebuilt_whl: - type: whl - files: - - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl + my_prebuilt_whl: + type: whl + files: + - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl resources: jobs: @@ -15,4 +15,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl + - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/nil_artifacts/databricks.yml b/acceptance/bundle/artifacts/nil_artifacts/databricks.yml new file mode 100644 index 0000000000..28e3acebda --- /dev/null +++ b/acceptance/bundle/artifacts/nil_artifacts/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: nil_artifacts_test + +artifacts: + # This reproduces the issue from GitHub #3017 + # where artifacts have empty definitions (only comments) + my_artifact: + # build: + # path: + # build: diff --git a/acceptance/bundle/artifacts/nil_artifacts/output.txt b/acceptance/bundle/artifacts/nil_artifacts/output.txt new file mode 100644 index 0000000000..f4bf0a6f40 --- /dev/null +++ b/acceptance/bundle/artifacts/nil_artifacts/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle validate +Error: Artifact not properly configured + in databricks.yml:7:15 + +please specify artifact properties + +Name: nil_artifacts_test +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/nil_artifacts_test/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/nil_artifacts/script b/acceptance/bundle/artifacts/nil_artifacts/script new file mode 100644 index 0000000000..f52b452ee6 --- /dev/null +++ b/acceptance/bundle/artifacts/nil_artifacts/script @@ -0,0 +1 @@ +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/artifacts/nil_artifacts/test.toml b/acceptance/bundle/artifacts/nil_artifacts/test.toml new file mode 100644 index 0000000000..3b639e6af6 --- /dev/null +++ b/acceptance/bundle/artifacts/nil_artifacts/test.toml @@ -0,0 +1,9 @@ +RecordRequests = false + +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml index d58674a64c..11eb80af82 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml +++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml @@ -9,8 +9,8 @@ variables: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/acceptance/bundle/artifacts/shell/bash/databricks.yml b/acceptance/bundle/artifacts/shell/bash/databricks.yml index eaf5495397..aae4c7b108 100644 --- a/acceptance/bundle/artifacts/shell/bash/databricks.yml +++ b/acceptance/bundle/artifacts/shell/bash/databricks.yml @@ -5,7 +5,7 @@ artifacts: my_artifact: executable: bash # echo in bash does not treat \n as a newline. - build: | + build: |- echo "\n\n\n should not see new lines" > out.shell.txt echo "multiline scripts should work" >> out.shell.txt pwd >> out.shell.txt diff --git a/acceptance/bundle/artifacts/shell/basic/databricks.yml b/acceptance/bundle/artifacts/shell/basic/databricks.yml index 68433d6ae0..947537ede2 100644 --- a/acceptance/bundle/artifacts/shell/basic/databricks.yml +++ b/acceptance/bundle/artifacts/shell/basic/databricks.yml @@ -3,6 +3,6 @@ bundle: artifacts: my_artifact: - build: | + build: |- echo "Hello, World!" > out.shell.txt echo "Bye, World!" >> out.shell.txt diff --git a/acceptance/bundle/artifacts/shell/cmd/databricks.yml b/acceptance/bundle/artifacts/shell/cmd/databricks.yml index 1161ddaa4e..732becc8fe 100644 --- a/acceptance/bundle/artifacts/shell/cmd/databricks.yml +++ b/acceptance/bundle/artifacts/shell/cmd/databricks.yml @@ -4,7 +4,7 @@ bundle: artifacts: my_artifact: executable: cmd - build: | + build: |- if defined CMDCMDLINE ( echo shell is cmd.exe> out.shell.txt ) diff --git a/acceptance/bundle/artifacts/shell/default/databricks.yml b/acceptance/bundle/artifacts/shell/default/databricks.yml index 2cbcb8f628..2cc7440c14 100644 --- a/acceptance/bundle/artifacts/shell/default/databricks.yml +++ b/acceptance/bundle/artifacts/shell/default/databricks.yml @@ -4,7 +4,7 @@ bundle: artifacts: my_artifact: # Default shell is bash. echo in bash does not treat \n as a newline. - build: | + build: |- echo "\n\n\n should not see new lines" > out.shell.txt echo "multiline scripts should work" >> out.shell.txt pwd >> out.shell.txt diff --git a/acceptance/bundle/artifacts/shell/sh/databricks.yml b/acceptance/bundle/artifacts/shell/sh/databricks.yml index babf01670a..64f2863a04 100644 --- a/acceptance/bundle/artifacts/shell/sh/databricks.yml +++ b/acceptance/bundle/artifacts/shell/sh/databricks.yml @@ -5,7 +5,7 @@ artifacts: my_artifact: executable: sh # echo in sh treats \n as a newline. - build: | + build: |- echo "\n\n\nshould see new lines" > out.shell.txt echo "multiline scripts should work" >> out.shell.txt pwd >> out.shell.txt diff --git a/acceptance/bundle/artifacts/unique_name_libraries/databricks.yml b/acceptance/bundle/artifacts/unique_name_libraries/databricks.yml index 930194ed44..0e22718f1b 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/databricks.yml +++ b/acceptance/bundle/artifacts/unique_name_libraries/databricks.yml @@ -6,8 +6,8 @@ variables: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/acceptance/bundle/artifacts/whl_change_version/.gitignore b/acceptance/bundle/artifacts/whl_change_version/.gitignore new file mode 100644 index 0000000000..f03e23bc26 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/acceptance/bundle/artifacts/whl_change_version/databricks.yml b/acceptance/bundle/artifacts/whl_change_version/databricks.yml new file mode 100644 index 0000000000..5a8d55ded0 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/databricks.yml @@ -0,0 +1,12 @@ +resources: + jobs: + test_job: + name: "[${bundle.target}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-aaaaa-bbbbbb" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_change_version/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_change_version/my_test_code/__init__.py new file mode 100644 index 0000000000..e96953330e --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/my_test_code/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.1.0" +__author__ = "Databricks" diff --git a/acceptance/bundle/artifacts/whl_change_version/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_change_version/my_test_code/__main__.py new file mode 100644 index 0000000000..ea918ce2d5 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/my_test_code/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print("Hello from my func") + print("Got arguments:") + print(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt new file mode 100644 index 0000000000..a2bfb79758 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -0,0 +1,138 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading dist/my_test_code-0.1.0-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +dist/my_test_code-0.1.0-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" + +>>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading dist/my_test_code-0.2.0-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 2 whl +dist/my_test_code-0.1.0-py3-none-any.whl +dist/my_test_code-0.2.0-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/reset +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" + +=== Restore config to target old wheel +>>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading dist/my_test_code-0.1.0-py3-none-any.whl... +Uploading dist/my_test_code-0.2.0-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 2 whl +dist/my_test_code-0.1.0-py3-none-any.whl +dist/my_test_code-0.2.0-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/reset +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script new file mode 100644 index 0000000000..2ccc104307 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -0,0 +1,39 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path out.requests.txt | grep import | sort + +rm out.requests.txt + + +trace update_file.py my_test_code/__init__.py 0.1.0 0.2.0 +trace $CLI bundle deploy + +trace find.py --expect 2 whl # there are now 2 wheels on disk + +title "Expecting 1 wheel in libraries section in /jobs/reset" +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path out.requests.txt | grep import | sort + +rm out.requests.txt + + +title 'Restore config to target old wheel' +trace update_file.py databricks.yml './dist/*.whl' './dist/my*0.1.0*.whl' +trace $CLI bundle deploy +trace find.py --expect 2 whl + +title "Expecting 1 wheel in libraries section in /jobs/reset" +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path out.requests.txt | grep import | sort + +rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_change_version/setup.py b/acceptance/bundle/artifacts/whl_change_version/setup.py new file mode 100644 index 0000000000..7a1317b2f5 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_change_version/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import my_test_code + +setup( + name="my_test_code", + version=my_test_code.__version__, + author=my_test_code.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["my_test_code"]), + entry_points={"group_1": "run=my_test_code.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/acceptance/bundle/artifacts/whl_dbfs/databricks.yml b/acceptance/bundle/artifacts/whl_dbfs/databricks.yml index 78519606f7..8b5ca158cf 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/databricks.yml +++ b/acceptance/bundle/artifacts/whl_dbfs/databricks.yml @@ -9,4 +9,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: dbfs:/path/to/dist/mywheel.whl + - whl: dbfs:/path/to/dist/mywheel.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml index 3225a9263b..bbfdee31a8 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml +++ b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml @@ -8,7 +8,7 @@ artifacts: my_prebuilt_whl: type: whl files: - - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl + - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl dynamic_version: true resources: @@ -16,32 +16,32 @@ resources: test_job: name: "[${bundle.target}] My Wheel Job" tasks: - - task_key: TestTask - existing_cluster_id: "0717-132531-5opeqon1" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: - - whl: ./my_test_code/dist/*.whl - - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl - for_each_task: - inputs: "[1]" - task: - task_key: SubTask - existing_cluster_id: "0717-132531-5opeqon1" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: - whl: ./my_test_code/dist/*.whl - - task_key: ServerlessTestTask - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - environment_key: "test_env" + - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl + for_each_task: + inputs: "[1]" + task: + task_key: SubTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl + - task_key: ServerlessTestTask + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + environment_key: "test_env" environments: - - environment_key: "test_env" - spec: - client: "1" - dependencies: - - ./my_test_code/dist/*.whl + - environment_key: "test_env" + spec: + client: "1" + dependencies: + - ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_explicit/databricks.yml b/acceptance/bundle/artifacts/whl_explicit/databricks.yml index 8173a1fedd..38c4a5b7cd 100644 --- a/acceptance/bundle/artifacts/whl_explicit/databricks.yml +++ b/acceptance/bundle/artifacts/whl_explicit/databricks.yml @@ -1,9 +1,9 @@ artifacts: - my_test_code: - type: whl - path: "./my_test_code" - # using 'python' there because 'python3' does not exist in virtualenv on windows - build: python setup.py bdist_wheel + my_test_code: + type: whl + path: "./my_test_code" + # using 'python' there because 'python3' does not exist in virtualenv on windows + build: python setup.py bdist_wheel resources: jobs: @@ -16,4 +16,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ./my_test_code/dist/*.whl + - whl: ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_implicit/databricks.yml b/acceptance/bundle/artifacts/whl_implicit/databricks.yml index 61019c57bb..5a8d55ded0 100644 --- a/acceptance/bundle/artifacts/whl_implicit/databricks.yml +++ b/acceptance/bundle/artifacts/whl_implicit/databricks.yml @@ -9,4 +9,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ./dist/*.whl + - whl: ./dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml b/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml index fb8999492b..37a0be7824 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml @@ -12,4 +12,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ./package/*.whl + - whl: ./package/*.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml b/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml index 17cceca0d7..ef7b0d4ad1 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml @@ -3,9 +3,9 @@ resources: test_job: name: "[${bundle.target}] My Wheel Job" tasks: - - task_key: TestTask - existing_cluster_id: "0717-aaaaa-bbbbbb" - notebook_task: - notebook_path: "/notebook.py" - libraries: - - whl: ./dist/*.whl + - task_key: TestTask + existing_cluster_id: "0717-aaaaa-bbbbbb" + notebook_task: + notebook_path: "/notebook.py" + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_multiple/databricks.yml b/acceptance/bundle/artifacts/whl_multiple/databricks.yml index e10933d5b2..072786d9ae 100644 --- a/acceptance/bundle/artifacts/whl_multiple/databricks.yml +++ b/acceptance/bundle/artifacts/whl_multiple/databricks.yml @@ -1,12 +1,12 @@ artifacts: - my_test_code: - type: whl - path: "./my_test_code" - build: "python setup.py bdist_wheel" - my_test_code_2: - type: whl - path: "./my_test_code" - build: "python setup2.py bdist_wheel" + my_test_code: + type: whl + path: "./my_test_code" + build: "python setup.py bdist_wheel" + my_test_code_2: + type: whl + path: "./my_test_code" + build: "python setup2.py bdist_wheel" resources: jobs: @@ -19,4 +19,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ./my_test_code/dist/*.whl + - whl: ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/databricks.yml b/acceptance/bundle/artifacts/whl_no_cleanup/databricks.yml index e6b0e4ddc7..a1540d407d 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/databricks.yml +++ b/acceptance/bundle/artifacts/whl_no_cleanup/databricks.yml @@ -12,4 +12,4 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ./dist/*.whl + - whl: ./dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml index d099684cfb..d5499c7c4f 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml @@ -9,5 +9,5 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ./dist/*.whl - - whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl + - whl: ./dist/*.whl + - whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/this_dab/databricks.yml b/acceptance/bundle/artifacts/whl_prebuilt_outside/this_dab/databricks.yml index ef500a9257..e21ddd7e35 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/this_dab/databricks.yml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/this_dab/databricks.yml @@ -3,9 +3,9 @@ bundle: sync: paths: - - ../other_dab + - ../other_dab exclude: - - ../other_dab/** + - ../other_dab/** resources: jobs: @@ -18,5 +18,5 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ../other_dab/dist/*.whl - - whl: ../other_dab/dist/lib/other_test_code-0.0.1-py3-none-any.whl + - whl: ../other_dab/dist/*.whl + - whl: ../other_dab/dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/this_dab/databricks.yml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/this_dab/databricks.yml index b7d82034fc..fc60fb3ce2 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/this_dab/databricks.yml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/this_dab/databricks.yml @@ -15,9 +15,9 @@ artifacts: sync: paths: - - ../other_dab + - ../other_dab exclude: - - ../other_dab/** + - ../other_dab/** resources: jobs: @@ -30,17 +30,17 @@ resources: package_name: "my_test_code" entry_point: "run" libraries: - - whl: ../other_dab/dist/*.whl - - whl: ../other_dab/dist/lib/other_test_code-0.0.1-py3-none-any.whl + - whl: ../other_dab/dist/*.whl + - whl: ../other_dab/dist/lib/other_test_code-0.0.1-py3-none-any.whl - task_key: ServerlessTestTask python_wheel_task: package_name: "my_test_code" entry_point: "run" - environment_key: "test_env" + environment_key: "test_env" environments: - - environment_key: "test_env" + - environment_key: "test_env" spec: client: "1" dependencies: - - ../other_dab/dist/*.whl - - ../other_dab/dist/lib/other_test_code-0.0.1-py3-none-any.whl + - ../other_dab/dist/*.whl + - ../other_dab/dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml index bf086e93cc..a7e54391f7 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml @@ -1,8 +1,8 @@ artifacts: - my_test_code: - type: whl - path: "./my_test_code" - build: python setup.py bdist_wheel + my_test_code: + type: whl + path: "./my_test_code" + build: python setup.py bdist_wheel resources: jobs: @@ -14,9 +14,9 @@ resources: python_wheel_task: package_name: "my_test_code" entry_point: "run" - environment_key: "test_env" + environment_key: "test_env" environments: - - environment_key: "test_env" + - environment_key: "test_env" spec: client: "1" dependencies: diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt index ea082fac30..9e72c92fe2 100644 --- a/acceptance/bundle/debug/out.stderr.txt +++ b/acceptance/bundle/debug/out.stderr.txt @@ -44,6 +44,7 @@ 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources) 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths +10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=TranslatePathsDashboards 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load) 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(init) 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources) diff --git a/acceptance/bundle/deploy/dashboard/nested-folders/dashboards/sample-dashboard.lvdash.json b/acceptance/bundle/deploy/dashboard/nested-folders/dashboards/sample-dashboard.lvdash.json new file mode 100644 index 0000000000..425cb33fb3 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/nested-folders/dashboards/sample-dashboard.lvdash.json @@ -0,0 +1 @@ +{"pages":[{"name":"02724bf2","displayName":"Dashboard test bundle-deploy-dashboard"}]} diff --git a/acceptance/bundle/deploy/dashboard/nested-folders/databricks.yml.tmpl b/acceptance/bundle/deploy/dashboard/nested-folders/databricks.yml.tmpl new file mode 100644 index 0000000000..87620709a3 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/nested-folders/databricks.yml.tmpl @@ -0,0 +1,10 @@ +# +# Acceptance test for deploying dashboards with the following setup: +# 1. dashboard file is inside the bundle root, but in a separate file from the .yml file +# 2. sync root is the same as the bundle root +# +bundle: + name: deploy-dashboard-nested-folders-$UNIQUE_NAME + +include: + - "resources/*.yml" diff --git a/acceptance/bundle/deploy/dashboard/nested-folders/output.txt b/acceptance/bundle/deploy/dashboard/nested-folders/output.txt new file mode 100644 index 0000000000..14f80ee007 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/nested-folders/output.txt @@ -0,0 +1,31 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-nested-folders-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] lakeview get [DASHBOARD_ID] +{ + "lifecycle_state": "ACTIVE", + "parent_path": "/Users/[USERNAME]", + "path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json", + "serialized_dashboard": { + "pages": [ + { + "name": "02724bf2", + "displayName": "Dashboard test bundle-deploy-dashboard", + "pageType": "PAGE_TYPE_CANVAS" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete dashboard dashboard1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-nested-folders-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/dashboard/nested-folders/resources/dashboards.yml.tmpl b/acceptance/bundle/deploy/dashboard/nested-folders/resources/dashboards.yml.tmpl new file mode 100644 index 0000000000..eaf846b128 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/nested-folders/resources/dashboards.yml.tmpl @@ -0,0 +1,8 @@ +resources: + dashboards: + dashboard1: + display_name: $DASHBOARD_DISPLAY_NAME + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + embed_credentials: true + file_path: ../dashboards/sample-dashboard.lvdash.json + parent_path: /Users/$CURRENT_USER_NAME diff --git a/acceptance/bundle/deploy/dashboard/nested-folders/script b/acceptance/bundle/deploy/dashboard/nested-folders/script new file mode 100644 index 0000000000..897a57b5c7 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/nested-folders/script @@ -0,0 +1,17 @@ +DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $(uuid)" +if [ -z "$CLOUD_ENV" ]; then + export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" +fi + +export DASHBOARD_DISPLAY_NAME +envsubst < databricks.yml.tmpl > databricks.yml +envsubst < resources/dashboards.yml.tmpl > resources/dashboards.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') +trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}' diff --git a/acceptance/bundle/deploy/dashboard/nested-folders/test.toml b/acceptance/bundle/deploy/dashboard/nested-folders/test.toml new file mode 100644 index 0000000000..00f60c8ef2 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/nested-folders/test.toml @@ -0,0 +1,18 @@ +Badness = "Cannot read dashboard, expecting deployment to succeed" +Local = true +Cloud = true +RequiresWarehouse = true + +Ignore = [ + "databricks.yml", + "resources/dashboards.yml", +] + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[DASHBOARD_ID]" + +[[Repls]] +# Windows: +Old = 'The system cannot find the file specified.' +New = 'no such file or directory' diff --git a/acceptance/bundle/deploy/dashboard/simple/databricks.yml.tmpl b/acceptance/bundle/deploy/dashboard/simple/databricks.yml.tmpl index 27b425c98a..682da0320b 100644 --- a/acceptance/bundle/deploy/dashboard/simple/databricks.yml.tmpl +++ b/acceptance/bundle/deploy/dashboard/simple/databricks.yml.tmpl @@ -1,3 +1,8 @@ +# +# Acceptance test for deploying dashboards with the following setup: +# 1. dashboard file is inside the bundle root +# 2. sync root is the same as the bundle root +# bundle: name: deploy-dashboard-test-$UNIQUE_NAME diff --git a/acceptance/bundle/deploy/dashboard/simple/output.txt b/acceptance/bundle/deploy/dashboard/simple/output.txt index e9ff6d057b..aca60647e4 100644 --- a/acceptance/bundle/deploy/dashboard/simple/output.txt +++ b/acceptance/bundle/deploy/dashboard/simple/output.txt @@ -10,7 +10,15 @@ Deployment complete! "lifecycle_state": "ACTIVE", "parent_path": "/Users/[USERNAME]", "path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json", - "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}" + "serialized_dashboard": { + "pages": [ + { + "name": "02724bf2", + "displayName": "Dashboard test bundle-deploy-dashboard", + "pageType": "PAGE_TYPE_CANVAS" + } + ] + } } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/dashboard/simple/script b/acceptance/bundle/deploy/dashboard/simple/script index 35aa749f11..dccb133141 100644 --- a/acceptance/bundle/deploy/dashboard/simple/script +++ b/acceptance/bundle/deploy/dashboard/simple/script @@ -1,6 +1,5 @@ DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $(uuid)" if [ -z "$CLOUD_ENV" ]; then - DASHBOARD_DISPLAY_NAME="test bundle/deploy/ 6260d50f-e8ff-4905-8f28-812345678903" # use hard-coded uuid when running locally export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" fi @@ -14,4 +13,4 @@ trap cleanup EXIT trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') -trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}' +trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}' diff --git a/acceptance/bundle/deploy/dashboard/simple/test.toml b/acceptance/bundle/deploy/dashboard/simple/test.toml index 26b6edd4c7..5f0a72c682 100644 --- a/acceptance/bundle/deploy/dashboard/simple/test.toml +++ b/acceptance/bundle/deploy/dashboard/simple/test.toml @@ -9,30 +9,3 @@ Ignore = [ [[Repls]] Old = "[0-9a-f]{32}" New = "[DASHBOARD_ID]" - -[[Server]] -Pattern = "POST /api/2.0/lakeview/dashboards" -Response.Body = ''' -{ - "dashboard_id":"1234567890abcdef1234567890abcdef" -} -''' - -[[Server]] -Pattern = "POST /api/2.0/lakeview/dashboards/{dashboard_id}/published" - -[[Server]] -Pattern = "GET /api/2.0/lakeview/dashboards/{dashboard_id}" -Response.Body = ''' -{ - "dashboard_id":"1234567890abcdef1234567890abcdef", - "display_name": "test dashboard 6260d50f-e8ff-4905-8f28-812345678903", - "lifecycle_state": "ACTIVE", - "path": "/Users/[USERNAME]/test bundle-deploy-dashboard 6260d50f-e8ff-4905-8f28-812345678903.lvdash.json", - "parent_path": "/Users/tester@databricks.com", - "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}" -} -''' - -[[Server]] -Pattern = "DELETE /api/2.0/lakeview/dashboards/{dashboard_id}" diff --git a/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/databricks.yml.tmpl b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/databricks.yml.tmpl new file mode 100644 index 0000000000..6b3de54b08 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/databricks.yml.tmpl @@ -0,0 +1,22 @@ +# +# Acceptance test for deploying dashboards with the following setup: +# 1. dashboard file is outside the bundle root +# 2. sync root is one level above bundle root +# +bundle: + name: deploy-dashboard-outside-bundle-root-$UNIQUE_NAME + +sync: + paths: + - .. + include: + - .. + +resources: + dashboards: + dashboard1: + display_name: $DASHBOARD_DISPLAY_NAME + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + embed_credentials: true + file_path: ../sample-dashboard.lvdash.json + parent_path: /Users/$CURRENT_USER_NAME diff --git a/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/output.txt b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/output.txt new file mode 100644 index 0000000000..e028e92101 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/output.txt @@ -0,0 +1,31 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-outside-bundle-root-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] lakeview get [DASHBOARD_ID] +{ + "lifecycle_state": "ACTIVE", + "parent_path": "/Users/[USERNAME]", + "path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json", + "serialized_dashboard": { + "pages": [ + { + "name": "02724bf2", + "displayName": "Dashboard test bundle-deploy-dashboard", + "pageType": "PAGE_TYPE_CANVAS" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete dashboard dashboard1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-outside-bundle-root-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/script b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/script new file mode 100644 index 0000000000..b84bef6328 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/script @@ -0,0 +1,17 @@ +DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $(uuid)" +if [ -z "$CLOUD_ENV" ]; then + export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" +fi +cp $TESTDIR/../simple/sample-dashboard.lvdash.json ../. + +export DASHBOARD_DISPLAY_NAME +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') +trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}' diff --git a/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/test.toml b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/test.toml new file mode 100644 index 0000000000..f540a35ba7 --- /dev/null +++ b/acceptance/bundle/deploy/dashboard/simple_outside_bundle_root/test.toml @@ -0,0 +1,17 @@ +Local = true +Cloud = true +RequiresWarehouse = true + +Ignore = [ + "databricks.yml", + "sample-dashboard.lvdash.json", +] + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[DASHBOARD_ID]" + +[[Repls]] +# Windows: +Old = 'The system cannot find the file specified.' +New = 'no such file or directory' diff --git a/acceptance/bundle/deploy/dashboard/simple_syncroot/databricks.yml.tmpl b/acceptance/bundle/deploy/dashboard/simple_syncroot/databricks.yml.tmpl index 833da56a6b..de130f6672 100644 --- a/acceptance/bundle/deploy/dashboard/simple_syncroot/databricks.yml.tmpl +++ b/acceptance/bundle/deploy/dashboard/simple_syncroot/databricks.yml.tmpl @@ -1,3 +1,8 @@ +# +# Acceptance test for deploying dashboards with the following setup: +# 1. dashboard file is within the bundle root +# 2. sync root is one level above bundle root +# bundle: name: deploy-dashboard-test-$UNIQUE_NAME diff --git a/acceptance/bundle/deploy/dashboard/simple_syncroot/output.txt b/acceptance/bundle/deploy/dashboard/simple_syncroot/output.txt index cd0f46dc7a..aca60647e4 100644 --- a/acceptance/bundle/deploy/dashboard/simple_syncroot/output.txt +++ b/acceptance/bundle/deploy/dashboard/simple_syncroot/output.txt @@ -1,9 +1,31 @@ >>> [CLI] bundle deploy -Error: failed to read serialized dashboard from file_path sample-dashboard.lvdash.json: open sample-dashboard.lvdash.json: no such file or directory +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +>>> [CLI] lakeview get [DASHBOARD_ID] +{ + "lifecycle_state": "ACTIVE", + "parent_path": "/Users/[USERNAME]", + "path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json", + "serialized_dashboard": { + "pages": [ + { + "name": "02724bf2", + "displayName": "Dashboard test bundle-deploy-dashboard", + "pageType": "PAGE_TYPE_CANVAS" + } + ] + } +} >>> [CLI] bundle destroy --auto-approve -Error: failed to read serialized dashboard from file_path sample-dashboard.lvdash.json: open sample-dashboard.lvdash.json: no such file or directory +The following resources will be deleted: + delete dashboard dashboard1 -Exit code: 1 +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/dashboard/simple_syncroot/script b/acceptance/bundle/deploy/dashboard/simple_syncroot/script index 4f5ff7805d..cf41bd69bf 100644 --- a/acceptance/bundle/deploy/dashboard/simple_syncroot/script +++ b/acceptance/bundle/deploy/dashboard/simple_syncroot/script @@ -1,6 +1,5 @@ DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $(uuid)" if [ -z "$CLOUD_ENV" ]; then - DASHBOARD_DISPLAY_NAME="test bundle/deploy/ 6260d50f-e8ff-4905-8f28-812345678903" # use hard-coded uuid when running locally export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" fi cp $TESTDIR/../simple/sample-dashboard.lvdash.json . @@ -16,4 +15,4 @@ trap cleanup EXIT trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') -trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}' +trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}' diff --git a/acceptance/bundle/deploy/dashboard/simple_syncroot/test.toml b/acceptance/bundle/deploy/dashboard/simple_syncroot/test.toml index 329f76ca71..f540a35ba7 100644 --- a/acceptance/bundle/deploy/dashboard/simple_syncroot/test.toml +++ b/acceptance/bundle/deploy/dashboard/simple_syncroot/test.toml @@ -1,4 +1,3 @@ -Badness = "Cannot read dashboard, expecting deployment to succeed" Local = true Cloud = true RequiresWarehouse = true @@ -16,30 +15,3 @@ New = "[DASHBOARD_ID]" # Windows: Old = 'The system cannot find the file specified.' New = 'no such file or directory' - -[[Server]] -Pattern = "POST /api/2.0/lakeview/dashboards" -Response.Body = ''' -{ - "dashboard_id":"1234567890abcdef1234567890abcdef" -} -''' - -[[Server]] -Pattern = "POST /api/2.0/lakeview/dashboards/{dashboard_id}/published" - -[[Server]] -Pattern = "GET /api/2.0/lakeview/dashboards/{dashboard_id}" -Response.Body = ''' -{ - "dashboard_id":"1234567890abcdef1234567890abcdef", - "display_name": "test dashboard 6260d50f-e8ff-4905-8f28-812345678903", - "lifecycle_state": "ACTIVE", - "path": "/Users/[USERNAME]/test bundle-deploy-dashboard 6260d50f-e8ff-4905-8f28-812345678903.lvdash.json", - "parent_path": "/Users/tester@databricks.com", - "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}" -} -''' - -[[Server]] -Pattern = "DELETE /api/2.0/lakeview/dashboards/{dashboard_id}" diff --git a/acceptance/bundle/deploy/experimental-python/databricks.yml b/acceptance/bundle/deploy/experimental-python/databricks.yml index e49852ae56..d2999290b1 100644 --- a/acceptance/bundle/deploy/experimental-python/databricks.yml +++ b/acceptance/bundle/deploy/experimental-python/databricks.yml @@ -1,4 +1,4 @@ -sync: { paths: [] } # dont need to copy files +sync: {paths: []} # dont need to copy files experimental: python: diff --git a/acceptance/bundle/deploy/fail-on-active-runs/databricks.yml b/acceptance/bundle/deploy/fail-on-active-runs/databricks.yml index 931e9b1bf2..ee4bd07d53 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/databricks.yml +++ b/acceptance/bundle/deploy/fail-on-active-runs/databricks.yml @@ -3,6 +3,6 @@ resources: my_job: name: My Job tasks: - - task_key: my_notebook - notebook_task: - notebook_path: my_notebook.py + - task_key: my_notebook + notebook_task: + notebook_path: my_notebook.py diff --git a/acceptance/bundle/deploy/python-notebook/databricks.yml b/acceptance/bundle/deploy/python-notebook/databricks.yml index 3a399779bc..eeccdd16c8 100644 --- a/acceptance/bundle/deploy/python-notebook/databricks.yml +++ b/acceptance/bundle/deploy/python-notebook/databricks.yml @@ -1,10 +1,10 @@ -sync: { paths: [] } # dont need to copy files +sync: {paths: []} # dont need to copy files resources: jobs: my_job: name: My Job tasks: - - task_key: my_notebook - notebook_task: - notebook_path: my_notebook.py + - task_key: my_notebook + notebook_task: + notebook_path: my_notebook.py diff --git a/acceptance/bundle/generate/dashboard-inplace/dash.lvdash.json b/acceptance/bundle/generate/dashboard-inplace/dash.lvdash.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/acceptance/bundle/generate/dashboard-inplace/dash.lvdash.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/generate/dashboard-inplace/databricks.yml b/acceptance/bundle/generate/dashboard-inplace/databricks.yml new file mode 100644 index 0000000000..5bbe24529b --- /dev/null +++ b/acceptance/bundle/generate/dashboard-inplace/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: dashboard update inplace + +resources: + dashboards: + test_dashboard: + display_name: "test dashboard" + warehouse_id: "" + file_path: ./dash.lvdash.json diff --git a/acceptance/bundle/generate/dashboard-inplace/output.txt b/acceptance/bundle/generate/dashboard-inplace/output.txt new file mode 100644 index 0000000000..29aa23a60a --- /dev/null +++ b/acceptance/bundle/generate/dashboard-inplace/output.txt @@ -0,0 +1,26 @@ + +>>> cat dash.lvdash.json +{} + +=== deploy initial dashboard +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard update inplace/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== update the dashboard +>>> [CLI] lakeview update [DASHBOARD_ID] --serialized-dashboard {"a":"b"} +{ + "etag":"[UUID]", + "serialized_dashboard":"{\"a\":\"b\"}" +} + +=== update the dashboard file using bundle generate +>>> [CLI] bundle generate dashboard --resource test_dashboard --force +Writing dashboard to "dash.lvdash.json" + +>>> cat dash.lvdash.json +{ + "a": "b" +} diff --git a/acceptance/bundle/generate/dashboard-inplace/script b/acceptance/bundle/generate/dashboard-inplace/script new file mode 100644 index 0000000000..f11a56da3c --- /dev/null +++ b/acceptance/bundle/generate/dashboard-inplace/script @@ -0,0 +1,13 @@ +trace cat dash.lvdash.json + +title "deploy initial dashboard" +trace $CLI bundle deploy +dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.test_dashboard.id') + +title "update the dashboard" +trace $CLI lakeview update $dashboard_id --serialized-dashboard '{"a":"b"}' + +title "update the dashboard file using bundle generate" +trace $CLI bundle generate dashboard --resource test_dashboard --force + +trace cat dash.lvdash.json diff --git a/acceptance/bundle/generate/dashboard-inplace/test.toml b/acceptance/bundle/generate/dashboard-inplace/test.toml new file mode 100644 index 0000000000..ab77a68240 --- /dev/null +++ b/acceptance/bundle/generate/dashboard-inplace/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[DASHBOARD_ID]" diff --git a/acceptance/bundle/generate/dashboard/dashboard.json b/acceptance/bundle/generate/dashboard/dashboard.json new file mode 100644 index 0000000000..502787ac8b --- /dev/null +++ b/acceptance/bundle/generate/dashboard/dashboard.json @@ -0,0 +1,5 @@ +{ + "display_name": "test dashboard", + "parent_path": "/test", + "serialized_dashboard": "{}" +} diff --git a/acceptance/bundle/generate/dashboard/databricks.yml b/acceptance/bundle/generate/dashboard/databricks.yml new file mode 100644 index 0000000000..726ee8a43a --- /dev/null +++ b/acceptance/bundle/generate/dashboard/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: dashboard-generate diff --git a/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json b/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/acceptance/bundle/generate/dashboard/out/dashboard/test_dashboard.lvdash.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/generate/dashboard/out/resource/test_dashboard.dashboard.yml b/acceptance/bundle/generate/dashboard/out/resource/test_dashboard.dashboard.yml new file mode 100644 index 0000000000..d282c46e69 --- /dev/null +++ b/acceptance/bundle/generate/dashboard/out/resource/test_dashboard.dashboard.yml @@ -0,0 +1,6 @@ +resources: + dashboards: + test_dashboard: + display_name: "test dashboard" + warehouse_id: "" + file_path: ../dashboard/test_dashboard.lvdash.json diff --git a/acceptance/bundle/generate/dashboard/output.txt b/acceptance/bundle/generate/dashboard/output.txt new file mode 100644 index 0000000000..f061923ac2 --- /dev/null +++ b/acceptance/bundle/generate/dashboard/output.txt @@ -0,0 +1,4 @@ + +>>> [CLI] bundle generate dashboard --existing-id [DASHBOARD_ID] --dashboard-dir out/dashboard --resource-dir out/resource +Writing dashboard to "out/dashboard/test_dashboard.lvdash.json" +Writing configuration to "out/resource/test_dashboard.dashboard.yml" diff --git a/acceptance/bundle/generate/dashboard/script b/acceptance/bundle/generate/dashboard/script new file mode 100644 index 0000000000..7a6616f57e --- /dev/null +++ b/acceptance/bundle/generate/dashboard/script @@ -0,0 +1,4 @@ +# create a dashboard to import +dashboard_id=$($CLI lakeview create --json @dashboard.json | jq -r '.dashboard_id') + +trace $CLI bundle generate dashboard --existing-id $dashboard_id --dashboard-dir out/dashboard --resource-dir out/resource diff --git a/acceptance/bundle/generate/dashboard/test.toml b/acceptance/bundle/generate/dashboard/test.toml new file mode 100644 index 0000000000..372aaafdc4 --- /dev/null +++ b/acceptance/bundle/generate/dashboard/test.toml @@ -0,0 +1,7 @@ +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[DASHBOARD_ID]" diff --git a/acceptance/bundle/includes/include_outside_root/databricks.yml b/acceptance/bundle/includes/include_outside_root/databricks.yml index 6addb7d2f8..36a72c08bf 100644 --- a/acceptance/bundle/includes/include_outside_root/databricks.yml +++ b/acceptance/bundle/includes/include_outside_root/databricks.yml @@ -2,4 +2,4 @@ bundle: name: include_outside_root include: - - a.yml + - a.yml diff --git a/acceptance/bundle/includes/non_yaml_in_include/databricks.yml b/acceptance/bundle/includes/non_yaml_in_include/databricks.yml index 162bd60133..15c68e1cf4 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/databricks.yml +++ b/acceptance/bundle/includes/non_yaml_in_include/databricks.yml @@ -2,5 +2,5 @@ bundle: name: non_yaml_in_includes include: - - test.py - - resources/*.yml + - test.py + - resources/*.yml diff --git a/acceptance/bundle/includes/non_yaml_in_include/output.txt b/acceptance/bundle/includes/non_yaml_in_include/output.txt index f5211cc4bd..c4f259cb08 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/output.txt +++ b/acceptance/bundle/includes/non_yaml_in_include/output.txt @@ -1,5 +1,5 @@ Error: Files in the 'include' configuration section must be YAML or JSON files. - in databricks.yml:5:4 + in databricks.yml:5:5 The file test.py in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead. diff --git a/acceptance/bundle/libraries/maven/databricks.yml b/acceptance/bundle/libraries/maven/databricks.yml index 785142626a..56ae0e0587 100644 --- a/acceptance/bundle/libraries/maven/databricks.yml +++ b/acceptance/bundle/libraries/maven/databricks.yml @@ -1,7 +1,6 @@ bundle: name: maven - resources: jobs: testjob: @@ -12,8 +11,8 @@ resources: main_class_name: com.databricks.example.Main libraries: - - maven: - coordinates: org.jsoup:jsoup:1.7.2 + - maven: + coordinates: org.jsoup:jsoup:1.7.2 new_cluster: spark_version: 15.4.x-scala2.12 @@ -21,7 +20,7 @@ resources: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/acceptance/bundle/libraries/pypi/databricks.yml b/acceptance/bundle/libraries/pypi/databricks.yml index 67f3da2543..d23f656377 100644 --- a/acceptance/bundle/libraries/pypi/databricks.yml +++ b/acceptance/bundle/libraries/pypi/databricks.yml @@ -1,7 +1,6 @@ bundle: name: pypi - resources: jobs: testjob: @@ -12,13 +11,13 @@ resources: project_directory: ./ profiles_directory: dbt_profiles/ commands: - - 'dbt deps --target=${bundle.target}' - - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + - 'dbt deps --target=${bundle.target}' + - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' libraries: - - pypi: - package: dbt-databricks>=1.8.0,<2.0.0 + - pypi: + package: dbt-databricks>=1.8.0,<2.0.0 new_cluster: spark_version: 15.4.x-scala2.12 @@ -26,7 +25,7 @@ resources: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/acceptance/bundle/local_state_staleness/databricks.yml.tmpl b/acceptance/bundle/local_state_staleness/databricks.yml.tmpl new file mode 100644 index 0000000000..7640c56341 --- /dev/null +++ b/acceptance/bundle/local_state_staleness/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: local-state-staleness-test + +workspace: + root_path: "~/.bundle/local-state-staleness-test-$UNIQUE_NAME" + +resources: + jobs: + test_job: + name: test-job-basic-$UNIQUE_NAME + tasks: + - task_key: my_notebook_task + new_cluster: + num_workers: 1 + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/local_state_staleness/hello_world.py b/acceptance/bundle/local_state_staleness/hello_world.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/local_state_staleness/hello_world.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/local_state_staleness/output.txt b/acceptance/bundle/local_state_staleness/output.txt new file mode 100644 index 0000000000..d2e5932d35 --- /dev/null +++ b/acceptance/bundle/local_state_staleness/output.txt @@ -0,0 +1,42 @@ + +=== Step 1: Deploy bundle A +>>> [CLI] bundle deploy --force-lock --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Step 2: Deploy bundle B +>>> [CLI] bundle deploy --force-lock --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Step 3: Deploy bundle A again (should use remote state) +>>> [CLI] bundle deploy --force-lock --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Step 4: Verify only one job exists +{ + "name": "test-job-basic-[UNIQUE_NAME]", + "id": [NUMID] +} + +=== Cleanup - destroy bundle A (bundle B does not have an active deployment) + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME] + +Deleting files... +Destroy complete! + +>>> rm -rf [TEST_TMP_DIR]/bundle_a + +>>> rm -rf [TEST_TMP_DIR]/bundle_b diff --git a/acceptance/bundle/local_state_staleness/script b/acceptance/bundle/local_state_staleness/script new file mode 100644 index 0000000000..6c434629e2 --- /dev/null +++ b/acceptance/bundle/local_state_staleness/script @@ -0,0 +1,44 @@ +# The approach for this test is as follows: +# 1) First deploy of bundle instance A +# 2) First deploy of bundle instance B +# 3) Second deploy of bundle instance A +# Because of deploy (2), the locally cached state of bundle instance A should be stale. +# Then for deploy (3), it must use the remote state over the stale local state. + +envsubst < databricks.yml.tmpl > databricks.yml + +# Create two separate bundle directories to simulate two bundle instances +BUNDLE_A_DIR="$TEST_TMP_DIR/bundle_a" +BUNDLE_B_DIR="$TEST_TMP_DIR/bundle_b" + +mkdir -p "$BUNDLE_A_DIR" "$BUNDLE_B_DIR" + +# Copy bundle files to both directories +cp databricks.yml hello_world.py "$BUNDLE_A_DIR/" +cp databricks.yml hello_world.py "$BUNDLE_B_DIR/" + +cleanup() { + title "Cleanup - destroy bundle A (bundle B does not have an active deployment)\n" + cd "$BUNDLE_A_DIR" || exit + trace "$CLI" bundle destroy --auto-approve + + trace rm -rf "$BUNDLE_A_DIR" + trace rm -rf "$BUNDLE_B_DIR" +} +trap cleanup EXIT + +title "Step 1: Deploy bundle A" +cd "$BUNDLE_A_DIR" +trace "$CLI" bundle deploy --force-lock --auto-approve + +title "Step 2: Deploy bundle B" +cd "$BUNDLE_B_DIR" +trace "$CLI" bundle deploy --force-lock --auto-approve + +title "Step 3: Deploy bundle A again (should use remote state)" +cd "$BUNDLE_A_DIR" +trace "$CLI" bundle deploy --force-lock --auto-approve + +title "Step 4: Verify only one job exists\n" +cd "$BUNDLE_A_DIR" +"$CLI" jobs list -o json | jq -r --arg name "test-job-basic-${UNIQUE_NAME}" '.[] | select(.settings.name == $name) | {name: .settings.name, id: .job_id}' diff --git a/acceptance/bundle/local_state_staleness/test.toml b/acceptance/bundle/local_state_staleness/test.toml new file mode 100644 index 0000000000..e93d23f722 --- /dev/null +++ b/acceptance/bundle/local_state_staleness/test.toml @@ -0,0 +1,2 @@ +Cloud = true +Local = true diff --git a/acceptance/bundle/paths/fallback_metric/resources/job.yml b/acceptance/bundle/paths/fallback_metric/resources/job.yml index 93e1d5fe4e..a345674ab6 100644 --- a/acceptance/bundle/paths/fallback_metric/resources/job.yml +++ b/acceptance/bundle/paths/fallback_metric/resources/job.yml @@ -10,8 +10,8 @@ resources: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode notebook_task: diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml index 6a94727dde..5ed46e048a 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml @@ -5,7 +5,7 @@ resources: pipelines: nyc_taxi_pipeline: libraries: - - notebook: { path: "${var.notebook_dir}/*.ipynb" } + - notebook: {path: "${var.notebook_dir}/*.ipynb"} variables: notebook_dir: diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml index d4a7e2b037..d3eb7eb94c 100644 --- a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml @@ -3,4 +3,4 @@ resources: nyc_taxi_pipeline: libraries: # path points to a notebook, not a file, it should error out - - file: { path: "../${var.notebook_dir}/*.py" } + - file: {path: "../${var.notebook_dir}/*.py"} diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml index 89e4d33ada..0d7ce2b1ba 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml @@ -3,16 +3,16 @@ resources: nyc_taxi_pipeline: libraries: # globs for notebooks and files are expanded - - notebook: { path: "../${var.notebook_dir}/*" } - - file: { path: "../${var.file_dir}/*" } + - notebook: {path: "../${var.notebook_dir}/*"} + - file: {path: "../${var.file_dir}/*"} # globs can include file extensions - - notebook: { path: "../${var.notebook_dir}/*.py" } - - file: { path: "../${var.file_dir}/*.py" } + - notebook: {path: "../${var.notebook_dir}/*.py"} + - file: {path: "../${var.file_dir}/*.py"} # non-glob files work - - notebook: { path: "../${var.notebook_dir}/nyc_taxi_loader.py" } + - notebook: {path: "../${var.notebook_dir}/nyc_taxi_loader.py"} # maven libraries and jars remain as-is - - maven: { coordinates: "org.jsoup:jsoup:1.7.2" } + - maven: {coordinates: "org.jsoup:jsoup:1.7.2"} - jar: "*/*.jar" # absolute paths and paths using URLs remain as-is - - notebook: { path: "/Workspace/Users/me@company.com/*.ipynb" } - - notebook: { path: "s3://notebooks/*.ipynb" } + - notebook: {path: "/Workspace/Users/me@company.com/*.ipynb"} + - notebook: {path: "s3://notebooks/*.ipynb"} diff --git a/acceptance/bundle/python/mutator-ordering/databricks.yml b/acceptance/bundle/python/mutator-ordering/databricks.yml index 8ff295f8c1..79ca24c848 100644 --- a/acceptance/bundle/python/mutator-ordering/databricks.yml +++ b/acceptance/bundle/python/mutator-ordering/databricks.yml @@ -1,13 +1,13 @@ bundle: name: my_project -sync: { paths: [] } # don't need to copy files +sync: {paths: []} # don't need to copy files experimental: python: mutators: - - "mutators:add_task_1" - - "mutators:add_task_2" + - "mutators:add_task_1" + - "mutators:add_task_2" resources: jobs: diff --git a/acceptance/bundle/python/pipelines-support/databricks.yml b/acceptance/bundle/python/pipelines-support/databricks.yml index ad22be89fa..2075b171f8 100644 --- a/acceptance/bundle/python/pipelines-support/databricks.yml +++ b/acceptance/bundle/python/pipelines-support/databricks.yml @@ -1,7 +1,7 @@ bundle: name: my_project -sync: { paths: [ ] } # don't need to copy files +sync: {paths: []} # don't need to copy files experimental: python: diff --git a/acceptance/bundle/python/resolve-variable/databricks.yml b/acceptance/bundle/python/resolve-variable/databricks.yml index d882ddf2cf..1775202701 100644 --- a/acceptance/bundle/python/resolve-variable/databricks.yml +++ b/acceptance/bundle/python/resolve-variable/databricks.yml @@ -1,7 +1,7 @@ bundle: name: my_project -sync: { paths: [] } # don't need to copy files +sync: {paths: []} # don't need to copy files experimental: python: @@ -20,9 +20,9 @@ variables: bool_variable_false: default: false list_variable: - default: [ 1, 2, 3 ] + default: [1, 2, 3] dict_variable: - default: { "a": 1, "b": 2 } + default: {"a": 1, "b": 2} complex_variable: default: task_key: "abc" diff --git a/acceptance/bundle/python/resource-loading/databricks.yml b/acceptance/bundle/python/resource-loading/databricks.yml index d8d26ecb04..b35128260a 100644 --- a/acceptance/bundle/python/resource-loading/databricks.yml +++ b/acceptance/bundle/python/resource-loading/databricks.yml @@ -1,13 +1,13 @@ bundle: name: my_project -sync: { paths: [] } # don't need to copy files +sync: {paths: []} # don't need to copy files experimental: python: resources: - - "resources:load_resources_1" - - "resources:load_resources_2" + - "resources:load_resources_1" + - "resources:load_resources_2" resources: jobs: diff --git a/acceptance/bundle/python/restricted-execution/databricks.yml b/acceptance/bundle/python/restricted-execution/databricks.yml index f50bc05600..cedcbf8e96 100644 --- a/acceptance/bundle/python/restricted-execution/databricks.yml +++ b/acceptance/bundle/python/restricted-execution/databricks.yml @@ -1,13 +1,12 @@ bundle: name: my_project -sync: { paths: [] } # don't need to copy files +sync: {paths: []} # don't need to copy files experimental: python: mutators: - - "mutators:read_envs" - + - "mutators:read_envs" resources: jobs: diff --git a/acceptance/bundle/python/unicode-support/databricks.yml b/acceptance/bundle/python/unicode-support/databricks.yml index 17a0838231..e50f29bab7 100644 --- a/acceptance/bundle/python/unicode-support/databricks.yml +++ b/acceptance/bundle/python/unicode-support/databricks.yml @@ -1,7 +1,7 @@ bundle: name: my_project -sync: { paths: [] } # don't need to copy files +sync: {paths: []} # don't need to copy files experimental: python: diff --git a/acceptance/bundle/quality_monitor/databricks.yml b/acceptance/bundle/quality_monitor/databricks.yml index 6138b9357c..b5ac063a0c 100644 --- a/acceptance/bundle/quality_monitor/databricks.yml +++ b/acceptance/bundle/quality_monitor/databricks.yml @@ -9,9 +9,9 @@ resources: output_schema_name: "main.dev" inference_log: granularities: ["1 day"] - timestamp_col: "timestamp" + timestamp_col: "timestamp" prediction_col: "prediction" - model_id_col: "model_id" + model_id_col: "model_id" problem_type: "PROBLEM_TYPE_REGRESSION" schedule: quartz_cron_expression: "0 0 12 * * ?" # every day at noon @@ -41,7 +41,7 @@ targets: output_schema_name: "main.prod" inference_log: granularities: ["1 hour"] - timestamp_col: "timestamp_prod" + timestamp_col: "timestamp_prod" prediction_col: "prediction_prod" - model_id_col: "model_id_prod" + model_id_col: "model_id_prod" problem_type: "PROBLEM_TYPE_REGRESSION" diff --git a/integration/bundle/bundles/clusters/template/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl similarity index 53% rename from integration/bundle/bundles/clusters/template/databricks.yml.tmpl rename to acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl index e0d6320a39..74c340a249 100644 --- a/integration/bundle/bundles/clusters/template/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/simple/databricks.yml.tmpl @@ -1,24 +1,24 @@ bundle: - name: basic + name: test-deploy-cluster-simple workspace: - root_path: "~/.bundle/{{.unique_id}}" + root_path: ~/.bundle/$UNIQUE_NAME resources: clusters: test_cluster: - cluster_name: "test-cluster-{{.unique_id}}" - spark_version: "{{.spark_version}}" - node_type_id: "{{.node_type_id}}" + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID num_workers: 2 spark_conf: "spark.executor.memory": "2g" jobs: foo: - name: test-job-with-cluster-{{.unique_id}} + name: test-job-with-cluster-$UNIQUE_NAME tasks: - - task_key: my_notebook_task + - task_key: my_spark_python_task existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" spark_python_task: python_file: ./hello_world.py diff --git a/integration/bundle/bundles/clusters/template/hello_world.py b/acceptance/bundle/resources/clusters/deploy/simple/hello_world.py similarity index 100% rename from integration/bundle/bundles/clusters/template/hello_world.py rename to acceptance/bundle/resources/clusters/deploy/simple/hello_world.py diff --git a/acceptance/bundle/resources/clusters/deploy/simple/output.txt b/acceptance/bundle/resources/clusters/deploy/simple/output.txt new file mode 100644 index 0000000000..3b648954b4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/simple/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Cluster should exist after bundle deployment: +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 2 +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete cluster test_cluster + delete job foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/clusters/deploy/simple/script b/acceptance/bundle/resources/clusters/deploy/simple/script new file mode 100644 index 0000000000..12977cedb0 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/simple/script @@ -0,0 +1,12 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Cluster should exist after bundle deployment:\n" +CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' diff --git a/acceptance/bundle/resources/clusters/deploy/simple/test.toml b/acceptance/bundle/resources/clusters/deploy/simple/test.toml new file mode 100644 index 0000000000..d075c79606 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/simple/test.toml @@ -0,0 +1,11 @@ +Local = false +Cloud = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt b/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt new file mode 100644 index 0000000000..34e222d414 --- /dev/null +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/output.txt @@ -0,0 +1,27 @@ + +>>> cp [TESTROOT]/bundle/resources/clusters/run/spark_python_task/../../deploy/simple/hello_world.py . + +>>> cp [TESTROOT]/bundle/resources/clusters/run/spark_python_task/../../deploy/simple/databricks.yml.tmpl . + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run foo +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "test-job-with-cluster-[UNIQUE_NAME]" RUNNING +[TIMESTAMP] "test-job-with-cluster-[UNIQUE_NAME]" TERMINATED SUCCESS +Hello World! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete cluster test_cluster + delete job foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/script b/acceptance/bundle/resources/clusters/run/spark_python_task/script new file mode 100644 index 0000000000..82408ac5d9 --- /dev/null +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/script @@ -0,0 +1,11 @@ +trace cp $TESTDIR/../../deploy/simple/hello_world.py . +trace cp $TESTDIR/../../deploy/simple/databricks.yml.tmpl . +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +trace $CLI bundle run foo diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml new file mode 100644 index 0000000000..51fc23573b --- /dev/null +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/test.toml @@ -0,0 +1,25 @@ +Local = false +CloudSlow = true +RecordRequests = false + +Ignore = [ + "databricks.yml", + "databricks.yml.tmpl", + "hello_world.py", +] + +[[Repls]] +Old = '2\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d' +New = "[TIMESTAMP]" + +[[Repls]] +Old = '\d{5,}' +New = "[NUMID]" + +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/run/basic/databricks.yml b/acceptance/bundle/run/basic/databricks.yml index c1761a8b97..2f787bb9e6 100644 --- a/acceptance/bundle/run/basic/databricks.yml +++ b/acceptance/bundle/run/basic/databricks.yml @@ -1,7 +1,6 @@ bundle: name: caterpillar - resources: jobs: foo: diff --git a/acceptance/bundle/state/databricks.yml b/acceptance/bundle/state/databricks.yml index 4775ba33d7..f423bf96a4 100644 --- a/acceptance/bundle/state/databricks.yml +++ b/acceptance/bundle/state/databricks.yml @@ -15,7 +15,7 @@ resources: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/output.txt b/acceptance/bundle/telemetry/deploy-artifact-path-type/output.txt index 4de0de5f79..a03920c3fd 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/output.txt @@ -11,98 +11,8 @@ Deployment complete! >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 2, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" } { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 2, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "UC_VOLUME" - } + "workspace_artifact_path_type": "UC_VOLUME" } diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/script b/acceptance/bundle/telemetry/deploy-artifact-path-type/script index 203e008267..d1a63928a6 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/script +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/script @@ -2,6 +2,6 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {workspace_artifact_path_type}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-compute-type/databricks.yml b/acceptance/bundle/telemetry/deploy-compute-type/databricks.yml index e764a56ed8..09ca30e83d 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/databricks.yml +++ b/acceptance/bundle/telemetry/deploy-compute-type/databricks.yml @@ -5,11 +5,11 @@ resources: jobs: my_job: environments: - - environment_key: "env" - spec: - client: "1" - dependencies: - - "test_package" + - environment_key: "env" + spec: + client: "1" + dependencies: + - "test_package" targets: one: @@ -31,15 +31,15 @@ targets: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode spark_python_task: python_file: ./test.py two: - resources: + resources: jobs: my_job: tasks: diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.telemetry.txt b/acceptance/bundle/telemetry/deploy-config-file-count/out.telemetry.txt deleted file mode 100644 index 3b06c63ebc..0000000000 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.telemetry.txt +++ /dev/null @@ -1,63 +0,0 @@ -{ - "frontend_log_event_id": "[UUID]", - "entry": { - "databricks_cli_log": { - "execution_context": { - "cmd_exec_id": "[CMD-EXEC-ID]", - "version": "[DEV_VERSION]", - "command": "bundle_deploy", - "operating_system": "[OS]", - "execution_time_ms": SMALL_INT, - "exit_code": 0 - }, - "bundle_deploy_event": { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 4, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 1, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } - } - } - } -} diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/output.txt b/acceptance/bundle/telemetry/deploy-config-file-count/output.txt index 7fe7e4837b..909e8d6c70 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/output.txt +++ b/acceptance/bundle/telemetry/deploy-config-file-count/output.txt @@ -5,3 +5,6 @@ Deploying resources... Deployment complete! >>> cat out.requests.txt +{ + "configuration_file_count": 4 +} diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/script b/acceptance/bundle/telemetry/deploy-config-file-count/script index 2d63d0641c..c495bdcb07 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/script +++ b/acceptance/bundle/telemetry/deploy-config-file-count/script @@ -1,9 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson' > out.telemetry.txt - -cmd_exec_id=$(extract_command_exec_id.py) - -update_file.py out.telemetry.txt $cmd_exec_id '[CMD-EXEC-ID]' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {configuration_file_count}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-error/out.telemetry.txt b/acceptance/bundle/telemetry/deploy-error/out.telemetry.txt index 7f8a122c1a..b3eff6a16f 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy-error/out.telemetry.txt @@ -1,30 +1,10 @@ { - "frontend_log_event_id": "[UUID]", - "entry": { - "databricks_cli_log": { - "execution_context": { - "cmd_exec_id": "[CMD-EXEC-ID]", - "version": "[DEV_VERSION]", - "command": "bundle_deploy", - "operating_system": "[OS]", - "execution_time_ms": SMALL_INT, - "exit_code": 1 - }, - "bundle_deploy_event": { - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0 - } - } + "execution_context": { + "cmd_exec_id": "[CMD-EXEC-ID]", + "version": "[DEV_VERSION]", + "command": "bundle_deploy", + "operating_system": "[OS]", + "execution_time_ms": SMALL_INT, + "exit_code": 1 } } diff --git a/acceptance/bundle/telemetry/deploy-error/script b/acceptance/bundle/telemetry/deploy-error/script index c14e81d75a..642cb0ef7d 100644 --- a/acceptance/bundle/telemetry/deploy-error/script +++ b/acceptance/bundle/telemetry/deploy-error/script @@ -1,6 +1,6 @@ errcode trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson' > out.telemetry.txt +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log | {execution_context}' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) diff --git a/acceptance/bundle/telemetry/deploy-mode/output.txt b/acceptance/bundle/telemetry/deploy-mode/output.txt index 3b7c64559c..29d39d59e7 100644 --- a/acceptance/bundle/telemetry/deploy-mode/output.txt +++ b/acceptance/bundle/telemetry/deploy-mode/output.txt @@ -17,98 +17,8 @@ A common practice is to use a username or principal name in this path, i.e. use >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 2, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "DEVELOPMENT", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "bundle_mode": "DEVELOPMENT" } { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 2, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "PRODUCTION", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "bundle_mode": "PRODUCTION" } diff --git a/acceptance/bundle/telemetry/deploy-mode/script b/acceptance/bundle/telemetry/deploy-mode/script index 8c1a180a67..f7257769ac 100644 --- a/acceptance/bundle/telemetry/deploy-mode/script +++ b/acceptance/bundle/telemetry/deploy-mode/script @@ -2,6 +2,6 @@ trace $CLI bundle deploy -t dev trace $CLI bundle deploy -t prod -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bundle_mode}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index 60111dfe21..350ba032bf 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -7,54 +7,30 @@ Deployment complete! >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 1, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 1, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 1, - "bool_values": [ - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "bool_values": [ + { + "key": "presets_name_prefix_is_set", + "value": true + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] } diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script index fe9af94278..67a3ba6299 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 4e20c480e5..11222ebafd 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -7,54 +7,30 @@ Deployment complete! >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 1, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 1, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 1, - "bool_values": [ - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "DEVELOPMENT", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "bool_values": [ + { + "key": "presets_name_prefix_is_set", + "value": true + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] } diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script index fe9af94278..67a3ba6299 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.telemetry.txt b/acceptance/bundle/telemetry/deploy-no-uuid/out.telemetry.txt index 2c288ea07b..d619909b9d 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.telemetry.txt @@ -1,70 +1,3 @@ { - "frontend_log_event_id": "[UUID]", - "entry": { - "databricks_cli_log": { - "execution_context": { - "cmd_exec_id": "[CMD-EXEC-ID]", - "version": "[DEV_VERSION]", - "command": "bundle_deploy", - "operating_system": "[OS]", - "execution_time_ms": SMALL_INT, - "exit_code": 0 - }, - "bundle_deploy_event": { - "bundle_uuid": "[ZERO_UUID]", - "deployment_id": "[UUID]", - "resource_count": 1, - "resource_job_count": 1, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "resource_job_ids": [ - "[NUMID]" - ], - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 1, - "bool_values": [ - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } - } - } - } + "bundle_uuid": "[ZERO_UUID]" } diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/script b/acceptance/bundle/telemetry/deploy-no-uuid/script index b82818d891..501b3dd76a 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/script +++ b/acceptance/bundle/telemetry/deploy-no-uuid/script @@ -1,10 +1,7 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson' > out.telemetry.txt +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event | {bundle_uuid}' > out.telemetry.txt -cmd_exec_id=$(extract_command_exec_id.py) - -update_file.py out.telemetry.txt $cmd_exec_id '[CMD-EXEC-ID]' update_file.py out.telemetry.txt "00000000-0000-0000-0000-000000000000" '[ZERO_UUID]' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-target-count/output.txt b/acceptance/bundle/telemetry/deploy-target-count/output.txt index f3e2dcf3fc..31581169f2 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/output.txt +++ b/acceptance/bundle/telemetry/deploy-target-count/output.txt @@ -6,50 +6,5 @@ Deployment complete! >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 0, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 3, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "target_count": 3 } diff --git a/acceptance/bundle/telemetry/deploy-target-count/script b/acceptance/bundle/telemetry/deploy-target-count/script index 9392350bf6..3022a2b5e4 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/script +++ b/acceptance/bundle/telemetry/deploy-target-count/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -t one -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {target_count}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-variable-count/output.txt b/acceptance/bundle/telemetry/deploy-variable-count/output.txt index 2f7dea0a0d..be4840e69e 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/output.txt +++ b/acceptance/bundle/telemetry/deploy-variable-count/output.txt @@ -6,50 +6,7 @@ Deployment complete! >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 6, - "complex_variable_count": 1, - "lookup_variable_count": 2, - "target_count": 1, - "bool_values": [ - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "variable_count": 6, + "lookup_variable_count": 2, + "complex_variable_count": 1 } diff --git a/acceptance/bundle/telemetry/deploy-variable-count/script b/acceptance/bundle/telemetry/deploy-variable-count/script index ff3f6db1b6..dad762899a 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/script +++ b/acceptance/bundle/telemetry/deploy-variable-count/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs.[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs.[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {variable_count, lookup_variable_count, complex_variable_count}' rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index eddd229c5f..941b1aa988 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -15,118 +15,70 @@ Deployment complete! >>> cat out.requests.txt { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 2, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 2, - "bool_values": [ - { - "key": "artifact_build_command_is_set", - "value": false - }, - { - "key": "artifact_files_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "bool_values": [ + { + "key": "artifact_build_command_is_set", + "value": false + }, + { + "key": "artifact_files_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] } { - "bundle_uuid": "[UUID]", - "deployment_id": "[UUID]", - "resource_count": 0, - "resource_job_count": 0, - "resource_pipeline_count": 0, - "resource_model_count": 0, - "resource_experiment_count": 0, - "resource_model_serving_endpoint_count": 0, - "resource_registered_model_count": 0, - "resource_quality_monitor_count": 0, - "resource_schema_count": 0, - "resource_volume_count": 0, - "resource_cluster_count": 0, - "resource_dashboard_count": 0, - "resource_app_count": 0, - "experimental": { - "configuration_file_count": 1, - "variable_count": 2, - "complex_variable_count": 0, - "lookup_variable_count": 0, - "target_count": 2, - "bool_values": [ - { - "key": "artifact_build_command_is_set", - "value": true - }, - { - "key": "artifact_files_is_set", - "value": true - }, - { - "key": "artifact_dynamic_version_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ], - "bundle_mode": "TYPE_UNSPECIFIED", - "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM" - } + "bool_values": [ + { + "key": "artifact_build_command_is_set", + "value": true + }, + { + "key": "artifact_files_is_set", + "value": true + }, + { + "key": "artifact_dynamic_version_is_set", + "value": true + }, + { + "key": "skip_artifact_cleanup", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] } diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/script b/acceptance/bundle/telemetry/deploy-whl-artifacts/script index bd048484e6..875167f8f2 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/script +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/script @@ -6,6 +6,6 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event' +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' rm out.requests.txt diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml index 4285a44eba..8b4eec8057 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml @@ -7,6 +7,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml # Deployment targets. # The default schema, catalog, etc. for dbt are defined in dbt_profiles/profiles.yml diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml index fdaf30dda2..32c56b6f23 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml @@ -1,38 +1,38 @@ # This file defines dbt profiles for deployed dbt jobs. my_dbt_sql: - target: dev # default target - outputs: - - # Doing local development with the dbt CLI? - # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' - # (See README.md) - - # The default target when deployed with the Databricks CLI - # N.B. when you use dbt from the command line, it uses the profile from .dbt/profiles.yml - dev: - type: databricks - method: http - catalog: main - schema: "{{ var('dev_schema') }}" - - http_path: /sql/2.0/warehouses/f00dcafe - - # The workspace host / token are provided by Databricks - # see databricks.yml for the workspace host used for 'dev' - host: "{{ env_var('DBT_HOST') }}" - token: "{{ env_var('DBT_ACCESS_TOKEN') }}" - - # The production target when deployed with the Databricks CLI - prod: - type: databricks - method: http - catalog: main - schema: default - - http_path: /sql/2.0/warehouses/f00dcafe - - # The workspace host / token are provided by Databricks - # see databricks.yml for the workspace host used for 'prod' - host: "{{ env_var('DBT_HOST') }}" - token: "{{ env_var('DBT_ACCESS_TOKEN') }}" + target: dev # default target + outputs: + + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) + + # The default target when deployed with the Databricks CLI + # N.B. when you use dbt from the command line, it uses the profile from .dbt/profiles.yml + dev: + type: databricks + method: http + catalog: main + schema: "{{ var('dev_schema') }}" + + http_path: /sql/2.0/warehouses/f00dcafe + + # The workspace host / token are provided by Databricks + # see databricks.yml for the workspace host used for 'dev' + host: "{{ env_var('DBT_HOST') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN') }}" + + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: main + schema: default + + http_path: /sql/2.0/warehouses/f00dcafe + + # The workspace host / token are provided by Databricks + # see databricks.yml for the workspace host used for 'prod' + host: "{{ env_var('DBT_HOST') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN') }}" diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml index 4218640d80..74044e47b7 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml @@ -15,7 +15,7 @@ seed-paths: ["src/seeds"] macro-paths: ["src/macros"] snapshot-paths: ["src/snapshots"] -clean-targets: # directories to be removed by `dbt clean` +clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index 368a148c3b..0211029c21 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -15,20 +15,19 @@ resources: tasks: - task_key: dbt - dbt_task: project_directory: ../ # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml profiles_directory: dbt_profiles/ commands: - # The dbt commands to run (see also dbt_profiles/profiles.yml; dev_schema is used in the dev profile) - - 'dbt deps --target=${bundle.target}' - - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + # The dbt commands to run (see also dbt_profiles/profiles.yml; dev_schema is used in the dev profile) + - 'dbt deps --target=${bundle.target}' + - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' libraries: - - pypi: - package: dbt-databricks>=1.8.0,<2.0.0 + - pypi: + package: dbt-databricks>=1.8.0,<2.0.0 new_cluster: spark_version: 15.4.x-scala2.12 @@ -36,7 +35,7 @@ resources: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 4d89b726a1..d05edefc70 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -37,8 +37,8 @@ + node_type_id: [NODE_TYPE_ID] + data_security_mode: SINGLE_USER + autoscale: -+ min_workers: 1 -+ max_workers: 4 ++ min_workers: 1 ++ max_workers: 4 --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml +++ output/my_default_python/resources/my_default_python.pipeline.yml @@ -4,8 +4,7 @@ diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml index 6080a368f9..91c01f25da 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml targets: dev: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml index 15ec4f0bfb..30b579f500 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml @@ -46,5 +46,5 @@ resources: node_type_id: [NODE_TYPE_ID] data_security_mode: SINGLE_USER autoscale: - min_workers: 1 - max_workers: 4 + min_workers: 1 + max_workers: 4 diff --git a/acceptance/bundle/templates/default-python/combinations/output.txt b/acceptance/bundle/templates/default-python/combinations/output.txt index 8715bcbb7f..adaec99e94 100644 --- a/acceptance/bundle/templates/default-python/combinations/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/output.txt @@ -9,6 +9,8 @@ Workspace to use (auto-detected, edit in 'X[UNIQUE_NAME]/databricks.yml'): [DATA Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +>>> yamlcheck.py + >>> [CLI] bundle validate -t dev Name: X[UNIQUE_NAME] Target: dev diff --git a/acceptance/bundle/templates/default-python/combinations/script b/acceptance/bundle/templates/default-python/combinations/script index 8e117f5cd2..ff1e5f4f00 100644 --- a/acceptance/bundle/templates/default-python/combinations/script +++ b/acceptance/bundle/templates/default-python/combinations/script @@ -2,6 +2,7 @@ envsubst < input.json.tmpl > input.json trace $CLI bundle init default-python --config-file ./input.json cd ./X* +trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml index 6080a368f9..91c01f25da 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml targets: dev: diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml index 07562ce7ad..e9d9093a96 100644 --- a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml # Variable declarations. These variables are assigned in the dev/prod targets below. variables: diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml index d336c9acfb..b09d99917e 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml @@ -24,6 +24,7 @@ artifacts: include: - resources/*.yml + - resources/*/*.yml targets: dev: diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.databricks.yml b/acceptance/bundle/templates/telemetry/dbt-sql/out.databricks.yml index e2aa7e9831..741b6d2b0a 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.databricks.yml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.databricks.yml @@ -7,6 +7,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml # Deployment targets. # The default schema, catalog, etc. for dbt are defined in dbt_profiles/profiles.yml diff --git a/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml b/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml index 80a431edc6..5adba2b4d8 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml +++ b/acceptance/bundle/templates/telemetry/default-python/out.databricks.yml @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml targets: dev: diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.databricks.yml b/acceptance/bundle/templates/telemetry/default-sql/out.databricks.yml index 49704391ea..608980ee90 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.databricks.yml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.databricks.yml @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml # Variable declarations. These variables are assigned in the dev/prod targets below. variables: diff --git a/acceptance/bundle/trampoline/warning_message/databricks.yml b/acceptance/bundle/trampoline/warning_message/databricks.yml index c6125f5f03..174dcc7bba 100644 --- a/acceptance/bundle/trampoline/warning_message/databricks.yml +++ b/acceptance/bundle/trampoline/warning_message/databricks.yml @@ -12,7 +12,6 @@ targets: interactive_cluster: spark_version: 14.2.x-cpu-ml-scala2.12 - resources: clusters: interactive_cluster: diff --git a/acceptance/bundle/variables/complex-cycle-self/output.txt b/acceptance/bundle/variables/complex-cycle-self/output.txt index 7447de349c..83ce6da568 100644 --- a/acceptance/bundle/variables/complex-cycle-self/output.txt +++ b/acceptance/bundle/variables/complex-cycle-self/output.txt @@ -1,4 +1,4 @@ -Warning: Detected unresolved variables after 11 resolution rounds +Warning: Variables references are too deep, stopping resolution after 11 rounds. Unresolved variables may remain. Name: cycle Target: default diff --git a/acceptance/bundle/variables/complex-cycle/output.txt b/acceptance/bundle/variables/complex-cycle/output.txt index 7447de349c..83ce6da568 100644 --- a/acceptance/bundle/variables/complex-cycle/output.txt +++ b/acceptance/bundle/variables/complex-cycle/output.txt @@ -1,4 +1,4 @@ -Warning: Detected unresolved variables after 11 resolution rounds +Warning: Variables references are too deep, stopping resolution after 11 rounds. Unresolved variables may remain. Name: cycle Target: default diff --git a/acceptance/bundle/variables/complex/databricks.yml b/acceptance/bundle/variables/complex/databricks.yml index 5dcc30b080..70813ce232 100644 --- a/acceptance/bundle/variables/complex/databricks.yml +++ b/acceptance/bundle/variables/complex/databricks.yml @@ -8,11 +8,11 @@ resources: - job_cluster_key: key new_cluster: ${var.cluster} tasks: - - task_key: test - job_cluster_key: key - libraries: ${variables.libraries.value} - # specific fields of complex variable are referenced: - task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}" + - task_key: test + job_cluster_key: key + libraries: ${variables.libraries.value} + # specific fields of complex variable are referenced: + task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}" variables: node_type: diff --git a/acceptance/bundle/variables/complex_multiple_files/databricks.yml b/acceptance/bundle/variables/complex_multiple_files/databricks.yml index a3249b9c1a..ef8037f8e7 100644 --- a/acceptance/bundle/variables/complex_multiple_files/databricks.yml +++ b/acceptance/bundle/variables/complex_multiple_files/databricks.yml @@ -37,7 +37,6 @@ variables: include: - ./variables/*.yml - targets: default: dev: diff --git a/acceptance/bundle/variables/issue_2436/databricks.yml b/acceptance/bundle/variables/issue_2436/databricks.yml index e47b7589b5..45fc94eb53 100644 --- a/acceptance/bundle/variables/issue_2436/databricks.yml +++ b/acceptance/bundle/variables/issue_2436/databricks.yml @@ -26,7 +26,6 @@ resources: job_two: tasks: ${var.job_tasks} - targets: dev: mode: development diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/databricks.yml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/databricks.yml new file mode 100644 index 0000000000..38f50f1832 --- /dev/null +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: issue-3039 + +variables: + tidal_service_account: + description: Gets tidal service account name/id. + lookup: + service_principal: "TIDALDBServAccount - ${var.uc_catalog}" + uc_catalog: + description: Unity Catalog prefix. + type: string + default: "" + +non_production_job_permissions: &non_prod_job_permissions + permissions: + - level: CAN_VIEW + service_principal_name: ${var.tidal_service_account} + +targets: + personal: + resources: + jobs: + xxx_job: + <<: *non_prod_job_permissions + variables: + uc_catalog: + description: Unity Catalog prefix. + default: "usdev" diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt b/acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt new file mode 100644 index 0000000000..0f85f8131e --- /dev/null +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/output.txt @@ -0,0 +1,11 @@ +Error: failed to resolve service-principal: TIDALDBServAccount - usdev, err: ServicePrincipal named 'TIDALDBServAccount - usdev' does not exist + +Name: issue-3039 +Target: personal +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/issue-3039/personal + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/script b/acceptance/bundle/variables/issue_3039_lookup_with_ref/script new file mode 100644 index 0000000000..72555b332a --- /dev/null +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/test.toml new file mode 100644 index 0000000000..030d24eb28 --- /dev/null +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/test.toml @@ -0,0 +1,23 @@ +[[Server]] +Pattern = "GET /api/2.0/preview/scim/v2/ServicePrincipals" +Response.Body = '''{}''' # this body causes error, but works to check if resolution is done correctly + +# For some reason this body causes SDK to loop forever +#Response.Body = '''{ +# "Resources": [ +# { +# "displayName": "TIDALDBServAccount - usdev", +# "groups": [], +# "id": "10000", +# "applicationId": "e700887d-8550-4667-8884-dbc94808cfb6", +# "schemas": [ +# "urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal" +# ], +# "active": true +# } +#], +#"totalResults": 1, +#"startIndex": 1, +#"itemsPerPage": 1, +#"schemas": [ +# "urn:ietf:params:scim:api:messages:2.0:ListResponse"]}''' diff --git a/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml b/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml index 4e52b5073c..3a26996dd8 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml +++ b/acceptance/bundle/variables/variable_overrides_in_target/databricks.yml @@ -9,8 +9,6 @@ resources: clusters: - num_workers: ${var.bar} - - variables: foo: default: "a_string" diff --git a/acceptance/cmd/workspace/apps/run-local/app/test.yml b/acceptance/cmd/workspace/apps/run-local/app/test.yml index 3d1c09a770..31de5261a3 100644 --- a/acceptance/cmd/workspace/apps/run-local/app/test.yml +++ b/acceptance/cmd/workspace/apps/run-local/app/test.yml @@ -1,4 +1,4 @@ command: -- python -- -c -- "print('Hello, world')" + - python + - -c + - "print('Hello, world')" diff --git a/acceptance/cmd/workspace/apps/run-local/app/value-from.yml b/acceptance/cmd/workspace/apps/run-local/app/value-from.yml index c5c47a9feb..9a6fe20d0f 100644 --- a/acceptance/cmd/workspace/apps/run-local/app/value-from.yml +++ b/acceptance/cmd/workspace/apps/run-local/app/value-from.yml @@ -1,7 +1,7 @@ command: -- python -- -c -- "print('Hello, world')" + - python + - -c + - "print('Hello, world')" env: - name: VALUE_FROM diff --git a/acceptance/dlt/install-dlt/databricks.yml b/acceptance/dlt/install-dlt/databricks.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/acceptance/dlt/install-dlt/output.txt b/acceptance/dlt/install-dlt/output.txt index a5075b1e88..4ae1319623 100644 --- a/acceptance/dlt/install-dlt/output.txt +++ b/acceptance/dlt/install-dlt/output.txt @@ -1,3 +1,14 @@ >>> errcode [CLI] install-dlt -dlt successfully installed +dlt successfully installed to the directory "[BUILD_DIR]" +dlt init is not yet implemented. This will initialize a new DLT project in the future. + +=== dlt file exists, should not overwrite +>>> errcode [CLI] install-dlt -d tmpdir +dlt successfully installed to the directory "tmpdir" +dlt init is not yet implemented. This will initialize a new DLT project in the future. +[DLT] -> [CLI] + +=== Executable not called as databricks +>>> errcode ./notdatabricks install-dlt +dlt successfully installed to the directory "[TEST_TMP_DIR]" diff --git a/acceptance/dlt/install-dlt/script b/acceptance/dlt/install-dlt/script index 208fd3b211..ddf6321431 100644 --- a/acceptance/dlt/install-dlt/script +++ b/acceptance/dlt/install-dlt/script @@ -1 +1,16 @@ trace errcode $CLI install-dlt +"$DLT" init + +title "dlt file exists, should not overwrite" +mkdir -p tmpdir +touch tmpdir/dlt +trace errcode $CLI install-dlt -d tmpdir +"$DLT" init +echo "$DLT -> $(realpath "$DLT")" +rm -rf tmpdir + +title "Executable not called as databricks" +cp $(command -v $CLI) notdatabricks +trace errcode ./notdatabricks install-dlt || true +rm -f notdatabricks +rm -f dlt diff --git a/acceptance/dlt/install-dlt/test.toml b/acceptance/dlt/install-dlt/test.toml new file mode 100644 index 0000000000..604105bb93 --- /dev/null +++ b/acceptance/dlt/install-dlt/test.toml @@ -0,0 +1,4 @@ +[[Repls]] +# Fix path with reverse slashes in the output for Windows. +Old = 'TEST_TMP_DIR]\\notdatabricks' +New = 'TEST_TMP_DIR]/notdatabricks' diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index 4f2f1163aa..cdfa4d5a82 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -44,6 +44,7 @@ Identity and Access Management Databricks SQL alerts The alerts API can be used to perform CRUD operations on alerts. alerts-legacy The alerts API can be used to perform CRUD operations on alerts. + alerts-v2 New version of SQL Alerts. dashboards In general, there is little need to modify dashboards using the API. data-sources This API is provided to assist you in making new query objects. queries The queries API can be used to perform CRUD operations on queries. @@ -123,6 +124,9 @@ Clean Rooms clean-room-task-runs Clean room task runs are the executions of notebooks in a clean room. clean-rooms A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each other’s data. +Quality Monitor v2 + quality-monitor-v2 Manage data quality of UC objects (currently support schema). + Additional Commands: account Databricks Account Commands api Perform Databricks API call diff --git a/acceptance/internal/handlers.go b/acceptance/internal/handlers.go index 3cf95165e1..85e0f46533 100644 --- a/acceptance/internal/handlers.go +++ b/acceptance/internal/handlers.go @@ -252,6 +252,23 @@ func addDefaultHandlers(server *testserver.Server) { } }) + // Dashboards: + server.Handle("GET", "/api/2.0/lakeview/dashboards/{dashboard_id}", func(req testserver.Request) any { + return testserver.MapGet(req.Workspace, req.Workspace.Dashboards, req.Vars["dashboard_id"]) + }) + server.Handle("POST", "/api/2.0/lakeview/dashboards", func(req testserver.Request) any { + return req.Workspace.DashboardCreate(req) + }) + server.Handle("POST", "/api/2.0/lakeview/dashboards/{dashboard_id}/published", func(req testserver.Request) any { + return req.Workspace.DashboardPublish(req) + }) + server.Handle("PATCH", "/api/2.0/lakeview/dashboards/{dashboard_id}", func(req testserver.Request) any { + return req.Workspace.DashboardUpdate(req) + }) + server.Handle("DELETE", "/api/2.0/lakeview/dashboards/{dashboard_id}", func(req testserver.Request) any { + return testserver.MapDelete(req.Workspace, req.Workspace.Dashboards, req.Vars["dashboard_id"]) + }) + // Pipelines: server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index ff30e10fbd..e062ee0b17 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -38,8 +38,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { }) } - cleanupPythonDistBuild(ctx, b) - for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) { a := b.Config.Artifacts[artifactName] diff --git a/bundle/artifacts/cleanup.go b/bundle/artifacts/cleanup.go deleted file mode 100644 index 9937178206..0000000000 --- a/bundle/artifacts/cleanup.go +++ /dev/null @@ -1,38 +0,0 @@ -package artifacts - -import ( - "context" - "os" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/python" - "github.com/databricks/cli/libs/utils" -) - -func cleanupPythonDistBuild(ctx context.Context, b *bundle.Bundle) { - removeFolders := make(map[string]bool, len(b.Config.Artifacts)) - cleanupWheelFolders := make(map[string]bool, len(b.Config.Artifacts)) - - for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) { - artifact := b.Config.Artifacts[artifactName] - if artifact.Type == "whl" && artifact.BuildCommand != "" { - dir := artifact.Path - removeFolders[filepath.Join(dir, "dist")] = true - cleanupWheelFolders[dir] = true - } - } - - for _, dir := range utils.SortedKeys(removeFolders) { - err := os.RemoveAll(dir) - if err != nil { - log.Infof(ctx, "Failed to remove %s: %s", dir, err) - } - } - - for _, dir := range utils.SortedKeys(cleanupWheelFolders) { - log.Infof(ctx, "Cleaning up Python build artifacts in %s", dir) - python.CleanupWheelFolder(dir) - } -} diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 1d9e188a0c..af358d3dda 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/patchwheel" ) func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { @@ -72,6 +73,12 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return v, nil } + // Note, we're applying this for all artifact types, not just "whl". + // Rationale: + // 1. type is optional + // 2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say. + matches = patchwheel.FilterLatestWheels(ctx, matches) + if len(matches) == 1 && matches[0] == source { // No glob expansion was performed. // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below. diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index 2bd0a8ba24..3bca30a805 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/python" "github.com/databricks/cli/libs/utils" @@ -35,6 +36,16 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) { artifact := b.Config.Artifacts[artifactName] + if artifact == nil { + l := b.Config.GetLocation("artifacts." + artifactName) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Artifact not properly configured", + Detail: "please specify artifact properties", + Locations: []dyn.Location{l}, + }) + continue + } b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "") b.Metrics.AddBoolValue(metrics.ArtifactFilesIsSet, len(artifact.Files) != 0) diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index ea29cc71d4..a3c99e5bf3 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -66,11 +66,16 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato } func ResolveVariableReferencesInLookup() bundle.Mutator { - return &resolveVariableReferences{prefixes: []string{ - "bundle", - "workspace", - "variables", - }, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), lookupFn: lookupForVariables} + return &resolveVariableReferences{ + prefixes: []string{ + "bundle", + "workspace", + "variables", + }, + pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), + lookupFn: lookupForVariables, + extraRounds: maxResolutionRounds - 1, + } } func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { @@ -149,7 +154,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) if round >= maxRounds-1 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, - Summary: fmt.Sprintf("Detected unresolved variables after %d resolution rounds", round+1), + Summary: fmt.Sprintf("Variables references are too deep, stopping resolution after %d rounds. Unresolved variables may remain.", round+1), // Would be nice to include names of the variables there, but that would complicate things more }) break diff --git a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go index 92d12ed893..27b55403d8 100644 --- a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go +++ b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go @@ -49,13 +49,7 @@ func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundl return dyn.InvalidValue, fmt.Errorf("failed to read serialized dashboard from file_path %s: %w", path, err) } - v, err = dyn.Set(v, serializedDashboardFieldName, dyn.V(string(contents))) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to set serialized_dashboard: %w", err) - } - - // Drop the "file_path" field. It is mutually exclusive with "serialized_dashboard". - return dyn.DropKeys(v, []string{filePathFieldName}) + return dyn.Set(v, serializedDashboardFieldName, dyn.V(string(contents))) }) }) diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go index a89de3c0cc..38f835793e 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/patchwheel" ) type expandPipelineGlobPaths struct{} @@ -17,7 +18,7 @@ func ExpandPipelineGlobPaths() bundle.Mutator { return &expandPipelineGlobPaths{} } -func (m *expandPipelineGlobPaths) expandLibrary(dir string, v dyn.Value) ([]dyn.Value, error) { +func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, v dyn.Value) ([]dyn.Value, error) { // Probe for the path field in the library. for _, p := range []dyn.Path{ dyn.NewPath(dyn.Key("notebook"), dyn.Key("path")), @@ -47,6 +48,8 @@ func (m *expandPipelineGlobPaths) expandLibrary(dir string, v dyn.Value) ([]dyn. return []dyn.Value{v}, nil } + matches = patchwheel.FilterLatestWheels(ctx, matches) + // Emit a new value for each match. var ev []dyn.Value for _, match := range matches { @@ -69,7 +72,7 @@ func (m *expandPipelineGlobPaths) expandLibrary(dir string, v dyn.Value) ([]dyn. return []dyn.Value{v}, nil } -func (m *expandPipelineGlobPaths) expandSequence(dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) { +func (m *expandPipelineGlobPaths) expandSequence(ctx context.Context, dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) { s, ok := v.AsSequence() if !ok { return dyn.InvalidValue, fmt.Errorf("expected sequence, got %s", v.Kind()) @@ -77,7 +80,7 @@ func (m *expandPipelineGlobPaths) expandSequence(dir string, p dyn.Path, v dyn.V var vs []dyn.Value for _, sv := range s { - v, err := m.expandLibrary(dir, sv) + v, err := m.expandLibrary(ctx, dir, sv) if err != nil { return dyn.InvalidValue, err } @@ -88,7 +91,7 @@ func (m *expandPipelineGlobPaths) expandSequence(dir string, p dyn.Path, v dyn.V return dyn.NewValue(vs, v.Locations()), nil } -func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { p := dyn.NewPattern( dyn.Key("resources"), @@ -99,7 +102,7 @@ func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) dia // Visit each pipeline's "libraries" field and expand any glob patterns. return dyn.MapByPattern(v, p, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { - return m.expandSequence(b.BundleRootPath, path, value) + return m.expandSequence(ctx, b.BundleRootPath, path, value) }) }) diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index cef71c62b3..f227bcfdf4 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -49,6 +49,11 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia "variables", ), mutator.NormalizePaths(), + + // Translate dashboard paths into paths in the workspace file system + // This must occur after NormalizePaths but before NormalizeAndInitializeResources, + // since the latter reads dashboard files and requires fully resolved paths. + mutator.TranslatePathsDashboards(), ) if diags.HasError() { return diags diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index ad707636df..f40e4c407c 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -49,15 +49,26 @@ func (err ErrIsNotNotebook) Error() string { type translatePaths struct{} +type translatePathsDashboards struct{} + // TranslatePaths converts paths to local notebook files into paths in the workspace file system. func TranslatePaths() bundle.Mutator { return &translatePaths{} } +// TranslatePathsDashboards converts paths to local dashboard files into paths in the workspace file system. +func TranslatePathsDashboards() bundle.Mutator { + return &translatePathsDashboards{} +} + func (m *translatePaths) Name() string { return "TranslatePaths" } +func (m *translatePathsDashboards) Name() string { + return "TranslatePathsDashboards" +} + // translateContext is a context for rewriting paths in a config. // It is freshly instantiated on every mutator apply call. // It provides access to the underlying bundle object such that @@ -281,12 +292,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V return dyn.NewValue(out, v.Locations()), nil } -func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - t := &translateContext{ - b: b, - seen: make(map[string]string), - } - +func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { // Set the remote root to the sync root if source-linked deployment is enabled. // Otherwise, set it to the workspace file path. if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) { @@ -297,13 +303,8 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { var err error - for _, fn := range []func(context.Context, dyn.Value) (dyn.Value, error){ - t.applyJobTranslations, - t.applyPipelineTranslations, - t.applyArtifactTranslations, - t.applyDashboardTranslations, - t.applyAppsTranslations, - } { + + for _, fn := range translations { v, err = fn(ctx, v) if err != nil { return dyn.InvalidValue, err @@ -315,6 +316,31 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return diag.FromErr(err) } +func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + t := &translateContext{ + b: b, + seen: make(map[string]string), + } + + return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + t.applyJobTranslations, + t.applyPipelineTranslations, + t.applyArtifactTranslations, + t.applyAppsTranslations, + }) +} + +func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + t := &translateContext{ + b: b, + seen: make(map[string]string), + } + + return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + t.applyDashboardTranslations, + }) +} + // gatherFallbackPaths collects the fallback paths for relative paths in the configuration. // Read more about the motivation for this functionality in the "fallback" path translation tests. func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) { diff --git a/bundle/config/resources/dashboard.go b/bundle/config/resources/dashboard.go index 572217baae..39b5a59abb 100644 --- a/bundle/config/resources/dashboard.go +++ b/bundle/config/resources/dashboard.go @@ -49,6 +49,8 @@ type Dashboard struct { EmbedCredentials bool `json:"embed_credentials,omitempty"` // FilePath points to the local `.lvdash.json` file containing the dashboard definition. + // This is inlined into serialized_dashboard during deployment. The file_path is kept around + // as metadata which is needed for `databricks bundle generate dashboard --resource ` to work. FilePath string `json:"file_path,omitempty"` } diff --git a/bundle/config/variable/resolve_metastore.go b/bundle/config/variable/resolve_metastore.go index 8a0a8c7edb..5460ccb3d3 100644 --- a/bundle/config/variable/resolve_metastore.go +++ b/bundle/config/variable/resolve_metastore.go @@ -2,8 +2,10 @@ package variable import ( "context" + "fmt" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/catalog" ) type resolveMetastore struct { @@ -11,11 +13,28 @@ type resolveMetastore struct { } func (l resolveMetastore) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) { - entity, err := w.Metastores.GetByName(ctx, l.name) + result, err := w.Metastores.ListAll(ctx, catalog.ListMetastoresRequest{}) if err != nil { return "", err } - return entity.MetastoreId, nil + + // Collect all metastores with the given name. + var entities []catalog.MetastoreInfo + for _, entity := range result { + if entity.Name == l.name { + entities = append(entities, entity) + } + } + + // Return the ID of the first matching metastore. + switch len(entities) { + case 0: + return "", fmt.Errorf("metastoren named %q does not exist", l.name) + case 1: + return entities[0].MetastoreId, nil + default: + return "", fmt.Errorf("there are %d instances of clusters named %q", len(entities), l.name) + } } func (l resolveMetastore) String() string { diff --git a/bundle/config/variable/resolve_metastore_test.go b/bundle/config/variable/resolve_metastore_test.go index 55c4d92d09..5d772e65bf 100644 --- a/bundle/config/variable/resolve_metastore_test.go +++ b/bundle/config/variable/resolve_metastore_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/assert" @@ -17,9 +16,9 @@ func TestResolveMetastore_ResolveSuccess(t *testing.T) { api := m.GetMockMetastoresAPI() api.EXPECT(). - GetByName(mock.Anything, "metastore"). - Return(&catalog.MetastoreInfo{ - MetastoreId: "abcd", + ListAll(mock.Anything, mock.Anything). + Return([]catalog.MetastoreInfo{ + {MetastoreId: "abcd", Name: "metastore"}, }, nil) ctx := context.Background() @@ -34,13 +33,15 @@ func TestResolveMetastore_ResolveNotFound(t *testing.T) { api := m.GetMockMetastoresAPI() api.EXPECT(). - GetByName(mock.Anything, "metastore"). - Return(nil, &apierr.APIError{StatusCode: 404}) + ListAll(mock.Anything, mock.Anything). + Return([]catalog.MetastoreInfo{ + {MetastoreId: "abcd", Name: "different"}, + }, nil) ctx := context.Background() l := resolveMetastore{name: "metastore"} _, err := l.Resolve(ctx, m.WorkspaceClient) - require.ErrorIs(t, err, apierr.ErrNotFound) + require.ErrorContains(t, err, "metastoren named \"metastore\" does not exist") } func TestResolveMetastore_String(t *testing.T) { diff --git a/bundle/deploy/terraform/tfdyn/convert_dashboard.go b/bundle/deploy/terraform/tfdyn/convert_dashboard.go index 8734565999..b51f6ae445 100644 --- a/bundle/deploy/terraform/tfdyn/convert_dashboard.go +++ b/bundle/deploy/terraform/tfdyn/convert_dashboard.go @@ -53,6 +53,12 @@ func convertDashboardResource(ctx context.Context, vin dyn.Value) (dyn.Value, er return dyn.InvalidValue, err } + // Drop the "file_path" field. It's always inlined into "serialized_dashboard". + vout, err = dyn.DropKeys(vout, []string{"file_path"}) + if err != nil { + return dyn.InvalidValue, err + } + return vout, nil } diff --git a/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go b/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go index 5d84949092..39b6f44714 100644 --- a/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go @@ -88,6 +88,7 @@ func TestConvertDashboardSerializedDashboardAny(t *testing.T) { }, }, }, + FilePath: "some/path/to/dashboard.lvdash.json", } vin, err := convert.FromTyped(src, dyn.NilValue) @@ -102,4 +103,7 @@ func TestConvertDashboardSerializedDashboardAny(t *testing.T) { assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{ "serialized_dashboard": `{"pages":[{"displayName":"New Page","layout":[]}]}`, }) + + // Assert that the "file_path" is dropped. + assert.NotContains(t, out.Dashboard["my_dashboard"], "file_path") } diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 395284c861..3f84637d46 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -534,6 +534,11 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "storage": "description": |- DBFS root directory for storing checkpoints and tables. + "tags": + "description": |- + A map of tags associated with the pipeline. + These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. + A maximum of 25 tags can be added to the pipeline. "target": "description": |- Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. @@ -1425,24 +1430,19 @@ github.com/databricks/databricks-sdk-go/service/compute.Environment: In this minimal environment spec, only pip dependencies are supported. "client": "description": |- - Client version used by the environment - The client is the user-facing environment of the runtime. - Each client comes with a specific set of pre-installed libraries. - The version is a string, consisting of the major client version. + Use `environment_version` instead. + "deprecation_message": |- + This field is deprecated "dependencies": "description": |- List of pip dependencies, as supported by the version of pip in this environment. - Each dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/ - Allowed dependency could be , , (WSFS or Volumes in Databricks), - E.g. dependencies: ["foo==0.0.1", "-r /Workspace/test/requirements.txt"] + Each dependency is a valid pip requirements file line per https://pip.pypa.io/en/stable/reference/requirements-file-format/. + Allowed dependencies include a requirement specifier, an archive URL, a local project path (such as WSFS or UC Volumes in Databricks), or a VCS project URL. "environment_version": "description": |- - We renamed `client` to `environment_version` in notebook exports. This field is meant solely so that imported notebooks with `environment_version` can be deserialized - correctly, in a backwards-compatible way (i.e. if `client` is specified instead of `environment_version`, it will be deserialized correctly). Do NOT use this field - for any other purpose, e.g. notebook storage. - This field is not yet exposed to customers (e.g. in the jobs API). - "x-databricks-preview": |- - PRIVATE + Required. Environment version used by the environment. + Each version comes with a specific Python version and a set of Python packages. + The version is a string, consisting of an integer. "jar_dependencies": "description": |- List of jar dependencies, should be string representing volume paths. For example: `/Volumes/path/to/test.jar`. @@ -1787,6 +1787,13 @@ github.com/databricks/databricks-sdk-go/service/jobs.DashboardTask: "description": |- Optional: The warehouse id to execute the dashboard with for the schedule. If not specified, the default warehouse of the dashboard will be used. +github.com/databricks/databricks-sdk-go/service/jobs.DbtCloudTask: + "connection_resource_name": + "description": |- + The resource name of the UC connection that authenticates the dbt Cloud for this task + "dbt_cloud_job_id": + "description": |- + Id of the dbt Cloud job to be triggered github.com/databricks/databricks-sdk-go/service/jobs.DbtTask: "catalog": "description": |- @@ -2540,6 +2547,11 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "dashboard_task": "description": |- The task refreshes a dashboard and sends a snapshot to subscribers. + "dbt_cloud_task": + "description": |- + Task type for dbt cloud + "x-databricks-preview": |- + PRIVATE "dbt_task": "description": |- The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. @@ -2878,6 +2890,8 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionSourceType: MANAGED_POSTGRESQL - |- ORACLE + - |- + TERADATA - |- SHAREPOINT - |- @@ -3692,9 +3706,15 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "instance_profile_arn": "description": |- ARN of the instance profile that the served entity uses to access AWS resources. + "max_provisioned_concurrency": + "description": |- + The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified. "max_provisioned_throughput": "description": |- The maximum tokens per second that the endpoint can scale up to. + "min_provisioned_concurrency": + "description": |- + The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified. "min_provisioned_throughput": "description": |- The minimum tokens per second that the endpoint can scale down to. @@ -3709,7 +3729,7 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: Whether the compute resources for the served entity should scale down to zero. "workload_size": "description": |- - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified. "workload_type": "description": |- The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). @@ -3720,9 +3740,15 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput: "instance_profile_arn": "description": |- ARN of the instance profile that the served entity uses to access AWS resources. + "max_provisioned_concurrency": + "description": |- + The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified. "max_provisioned_throughput": "description": |- The maximum tokens per second that the endpoint can scale up to. + "min_provisioned_concurrency": + "description": |- + The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified. "min_provisioned_throughput": "description": |- The minimum tokens per second that the endpoint can scale down to. @@ -3739,7 +3765,7 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput: Whether the compute resources for the served entity should scale down to zero. "workload_size": "description": |- - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified. "workload_type": "description": |- The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). diff --git a/bundle/internal/schema/testdata/pass/run_job_task.yml b/bundle/internal/schema/testdata/pass/run_job_task.yml index be2ca22cd6..0642e22d42 100644 --- a/bundle/internal/schema/testdata/pass/run_job_task.yml +++ b/bundle/internal/schema/testdata/pass/run_job_task.yml @@ -3,7 +3,6 @@ bundle: databricks_cli_version: 0.200.0 compute_id: "mycompute" - variables: simplevar: default: 5678 diff --git a/bundle/internal/tf/codegen/go.mod b/bundle/internal/tf/codegen/go.mod index 462ed2b673..e828f3b1ad 100644 --- a/bundle/internal/tf/codegen/go.mod +++ b/bundle/internal/tf/codegen/go.mod @@ -1,6 +1,7 @@ module github.com/databricks/cli/bundle/internal/tf/codegen go 1.24 + toolchain go1.24.2 require ( @@ -16,7 +17,7 @@ require ( require ( github.com/ProtonMail/go-crypto v1.1.3 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/cloudflare/circl v1.5.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect golang.org/x/crypto v0.35.0 // indirect diff --git a/bundle/internal/tf/codegen/go.sum b/bundle/internal/tf/codegen/go.sum index 9dd72ec6fb..b76bd745ca 100644 --- a/bundle/internal/tf/codegen/go.sum +++ b/bundle/internal/tf/codegen/go.sum @@ -6,8 +6,8 @@ github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXx github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= -github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= -github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 7a808f6270..039157b086 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/patchwheel" ) type expand struct{} @@ -37,7 +38,7 @@ func getLibDetails(v dyn.Value) (string, string, bool) { return "", "", false } -func findMatches(b *bundle.Bundle, path string) ([]string, error) { +func findMatches(ctx context.Context, b *bundle.Bundle, path string) ([]string, error) { matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, path)) if err != nil { return nil, err @@ -51,6 +52,8 @@ func findMatches(b *bundle.Bundle, path string) ([]string, error) { } } + matches = patchwheel.FilterLatestWheels(ctx, matches) + // We make the matched path relative to the sync root path before storing it // to allow upload mutator to distinguish between local and remote paths for i, match := range matches { @@ -69,7 +72,7 @@ func isGlobPattern(path string) bool { return strings.ContainsAny(path, "*?[") } -func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { +func expandLibraries(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { var output []dyn.Value var diags diag.Diagnostics @@ -84,7 +87,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic lp = lp.Append(dyn.Key(libType)) - matches, err := findMatches(b, path) + matches, err := findMatches(ctx, b, path) if err != nil { diags = diags.Append(matchError(lp, lib.Locations(), err.Error())) continue @@ -100,7 +103,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic return diags, output } -func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { +func expandEnvironmentDeps(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { var output []dyn.Value var diags diag.Diagnostics @@ -113,7 +116,7 @@ func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diag continue } - matches, err := findMatches(b, path) + matches, err := findMatches(ctx, b, path) if err != nil { diags = diags.Append(matchError(lp, dep.Locations(), err.Error())) continue @@ -129,7 +132,7 @@ func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diag type expandPattern struct { pattern dyn.Pattern - fn func(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) + fn func(ctx context.Context, b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) } var taskLibrariesPattern = dyn.NewPattern( @@ -184,7 +187,7 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var err error for _, expander := range expanders { v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { - d, output := expander.fn(b, p, lv) + d, output := expander.fn(ctx, b, p, lv) diags = diags.Extend(d) return dyn.V(output), nil }) diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index ae54e86570..022c71b599 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" ) @@ -23,11 +24,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions) (d }() diags = diags.Extend(bundle.ApplySeq(ctx, b, - terraform.StatePull(), + statemgmt.StatePull(), terraform.Interpolate(), terraform.Write(), terraform.Import(opts), - terraform.StatePush(), + statemgmt.StatePush(), )) return diags @@ -46,11 +47,11 @@ func Unbind(ctx context.Context, b *bundle.Bundle, resourceType, resourceKey str }() diags = diags.Extend(bundle.ApplySeq(ctx, b, - terraform.StatePull(), + statemgmt.StatePull(), terraform.Interpolate(), terraform.Write(), terraform.Unbind(resourceType, resourceKey), - terraform.StatePush(), + statemgmt.StatePush(), )) return diags diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index d196019bc4..e24f9102a8 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/bundle/trampoline" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" @@ -148,7 +149,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // following original logic, continuing with sequence below even if terraform had errors diags = diags.Extend(bundle.ApplySeq(ctx, b, - terraform.StatePush(), + statemgmt.StatePush(), terraform.Load(), apps.InterpolateVariables(), apps.UploadConfig(), @@ -185,7 +186,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand }() diags = bundle.ApplySeq(ctx, b, - terraform.StatePull(), + statemgmt.StatePull(), terraform.CheckDashboardsModifiedRemotely(), deploy.StatePull(), mutator.ValidateGitDetails(), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index b2c01343b5..9c2285dfca 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" @@ -116,7 +117,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) (diags diag.Diagnostics) { }() diags = diags.Extend(bundle.ApplySeq(ctx, b, - terraform.StatePull(), + statemgmt.StatePull(), terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("destroy")), diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index cf79d61226..5b220c5c0b 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1083,6 +1083,10 @@ "description": "DBFS root directory for storing checkpoints and tables.", "$ref": "#/$defs/string" }, + "tags": { + "description": "A map of tags associated with the pipeline.\nThese are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations.\nA maximum of 25 tags can be added to the pipeline.", + "$ref": "#/$defs/map/string" + }, "target": { "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", "$ref": "#/$defs/string" @@ -3325,18 +3329,18 @@ "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip dependencies are supported.", "properties": { "client": { - "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version.", - "$ref": "#/$defs/string" + "description": "Use `environment_version` instead.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true }, "dependencies": { "description": "List of pip dependencies, as supported by the version of pip in this environment.", "$ref": "#/$defs/slice/string" }, "environment_version": { - "description": "We renamed `client` to `environment_version` in notebook exports. This field is meant solely so that imported notebooks with `environment_version` can be deserialized\ncorrectly, in a backwards-compatible way (i.e. if `client` is specified instead of `environment_version`, it will be deserialized correctly). Do NOT use this field\nfor any other purpose, e.g. notebook storage.\nThis field is not yet exposed to customers (e.g. in the jobs API).", - "$ref": "#/$defs/string", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "description": "Required. Environment version used by the environment.\nEach version comes with a specific Python version and a set of Python packages.\nThe version is a string, consisting of an integer.", + "$ref": "#/$defs/string" }, "jar_dependencies": { "description": "List of jar dependencies, should be string representing volume paths. For example: `/Volumes/path/to/test.jar`.", @@ -3345,10 +3349,7 @@ "doNotSuggest": true } }, - "additionalProperties": false, - "required": [ - "client" - ] + "additionalProperties": false }, { "type": "string", @@ -4009,6 +4010,28 @@ } ] }, + "jobs.DbtCloudTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "connection_resource_name": { + "description": "The resource name of the UC connection that authenticates the dbt Cloud for this task", + "$ref": "#/$defs/string" + }, + "dbt_cloud_job_id": { + "description": "Id of the dbt Cloud job to be triggered", + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.DbtTask": { "oneOf": [ { @@ -5382,6 +5405,12 @@ "description": "The task refreshes a dashboard and sends a snapshot to subscribers.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DashboardTask" }, + "dbt_cloud_task": { + "description": "Task type for dbt cloud", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtCloudTask", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "dbt_task": { "description": "The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtTask" @@ -5982,6 +6011,7 @@ "SERVICENOW", "MANAGED_POSTGRESQL", "ORACLE", + "TERADATA", "SHAREPOINT", "DYNAMICS365" ] @@ -7412,10 +7442,18 @@ "description": "ARN of the instance profile that the served entity uses to access AWS resources.", "$ref": "#/$defs/string" }, + "max_provisioned_concurrency": { + "description": "The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, "max_provisioned_throughput": { "description": "The maximum tokens per second that the endpoint can scale up to.", "$ref": "#/$defs/int" }, + "min_provisioned_concurrency": { + "description": "The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, "min_provisioned_throughput": { "description": "The minimum tokens per second that the endpoint can scale down to.", "$ref": "#/$defs/int" @@ -7433,7 +7471,7 @@ "$ref": "#/$defs/bool" }, "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.", + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified.", "$ref": "#/$defs/string" }, "workload_type": { @@ -7462,10 +7500,18 @@ "description": "ARN of the instance profile that the served entity uses to access AWS resources.", "$ref": "#/$defs/string" }, + "max_provisioned_concurrency": { + "description": "The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, "max_provisioned_throughput": { "description": "The maximum tokens per second that the endpoint can scale up to.", "$ref": "#/$defs/int" }, + "min_provisioned_concurrency": { + "description": "The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, "min_provisioned_throughput": { "description": "The minimum tokens per second that the endpoint can scale down to.", "$ref": "#/$defs/int" @@ -7489,7 +7535,7 @@ "$ref": "#/$defs/bool" }, "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.", + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified.", "$ref": "#/$defs/string" }, "workload_type": { diff --git a/bundle/deploy/terraform/state_pull.go b/bundle/statemgmt/state_pull.go similarity index 91% rename from bundle/deploy/terraform/state_pull.go rename to bundle/statemgmt/state_pull.go index 4e1e2b1c57..777d770aa9 100644 --- a/bundle/deploy/terraform/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -1,4 +1,4 @@ -package terraform +package statemgmt import ( "context" @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy" + tf "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" ) @@ -34,7 +35,7 @@ func (l *statePull) remoteState(ctx context.Context, b *bundle.Bundle) (*tfState return nil, nil, err } - r, err := f.Read(ctx, TerraformStateFileName) + r, err := f.Read(ctx, tf.TerraformStateFileName) if err != nil { return nil, nil, err } @@ -55,12 +56,12 @@ func (l *statePull) remoteState(ctx context.Context, b *bundle.Bundle) (*tfState } func (l *statePull) localState(ctx context.Context, b *bundle.Bundle) (*tfState, error) { - dir, err := Dir(ctx, b) + dir, err := tf.Dir(ctx, b) if err != nil { return nil, err } - content, err := os.ReadFile(filepath.Join(dir, TerraformStateFileName)) + content, err := os.ReadFile(filepath.Join(dir, tf.TerraformStateFileName)) if err != nil { return nil, err } @@ -75,12 +76,12 @@ func (l *statePull) localState(ctx context.Context, b *bundle.Bundle) (*tfState, } func (l *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - dir, err := Dir(ctx, b) + dir, err := tf.Dir(ctx, b) if err != nil { return diag.FromErr(err) } - localStatePath := filepath.Join(dir, TerraformStateFileName) + localStatePath := filepath.Join(dir, tf.TerraformStateFileName) // Case: Remote state file does not exist. In this case we fallback to using the // local Terraform state. This allows users to change the "root_path" their bundle is diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/statemgmt/state_pull_test.go similarity index 97% rename from bundle/deploy/terraform/state_pull_test.go rename to bundle/statemgmt/state_pull_test.go index c4798c578f..9720ff09b5 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/statemgmt/state_pull_test.go @@ -1,4 +1,4 @@ -package terraform +package statemgmt import ( "bytes" @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + tf "github.com/databricks/cli/bundle/deploy/terraform" mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" "github.com/databricks/cli/libs/filer" "github.com/stretchr/testify/assert" @@ -24,7 +25,7 @@ func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) fi f := mockfiler.NewMockFiler(t) f. EXPECT(). - Read(mock.Anything, TerraformStateFileName). + Read(mock.Anything, tf.TerraformStateFileName). Return(io.NopCloser(bytes.NewReader(buf)), merr). Times(1) return f diff --git a/bundle/deploy/terraform/state_push.go b/bundle/statemgmt/state_push.go similarity index 82% rename from bundle/deploy/terraform/state_push.go rename to bundle/statemgmt/state_push.go index 6cdde13716..f1a4b9d368 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -1,4 +1,4 @@ -package terraform +package statemgmt import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy" + tf "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" @@ -29,13 +30,13 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic return diag.FromErr(err) } - dir, err := Dir(ctx, b) + dir, err := tf.Dir(ctx, b) if err != nil { return diag.FromErr(err) } // Expect the state file to live under dir. - local, err := os.Open(filepath.Join(dir, TerraformStateFileName)) + local, err := os.Open(filepath.Join(dir, tf.TerraformStateFileName)) if errors.Is(err, fs.ErrNotExist) { // The state file can be absent if terraform apply is skipped because // there are no changes to apply in the plan. @@ -50,7 +51,7 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic // Upload state file from local cache directory to filer. cmdio.LogString(ctx, "Updating deployment state...") log.Infof(ctx, "Writing local state file to remote state directory") - err = f.Write(ctx, TerraformStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) + err = f.Write(ctx, tf.TerraformStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/statemgmt/state_push_test.go similarity index 98% rename from bundle/deploy/terraform/state_push_test.go rename to bundle/statemgmt/state_push_test.go index e022dee1bd..88bad10918 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/statemgmt/state_push_test.go @@ -1,4 +1,4 @@ -package terraform +package statemgmt import ( "context" diff --git a/bundle/deploy/terraform/state_test.go b/bundle/statemgmt/state_test.go similarity index 87% rename from bundle/deploy/terraform/state_test.go rename to bundle/statemgmt/state_test.go index 73d7cb0dee..2405aa475b 100644 --- a/bundle/deploy/terraform/state_test.go +++ b/bundle/statemgmt/state_test.go @@ -1,4 +1,4 @@ -package terraform +package statemgmt import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy" + tf "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/libs/filer" "github.com/stretchr/testify/require" ) @@ -21,9 +22,9 @@ func identityFiler(f filer.Filer) deploy.FilerFactory { } func localStateFile(t *testing.T, ctx context.Context, b *bundle.Bundle) string { - dir, err := Dir(ctx, b) + dir, err := tf.Dir(ctx, b) require.NoError(t, err) - return filepath.Join(dir, TerraformStateFileName) + return filepath.Join(dir, tf.TerraformStateFileName) } func readLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle) map[string]any { diff --git a/bundle/tests/environment_key_only/databricks.yml b/bundle/tests/environment_key_only/databricks.yml index caa34f8e35..7dc02d441a 100644 --- a/bundle/tests/environment_key_only/databricks.yml +++ b/bundle/tests/environment_key_only/databricks.yml @@ -11,6 +11,6 @@ resources: python_wheel_task: package_name: "my_test_code" entry_point: "run" - environment_key: "test_env" + environment_key: "test_env" environments: - - environment_key: "test_env" + - environment_key: "test_env" diff --git a/bundle/tests/job_cluster_key/databricks.yml b/bundle/tests/job_cluster_key/databricks.yml index bd863db3e5..1574366717 100644 --- a/bundle/tests/job_cluster_key/databricks.yml +++ b/bundle/tests/job_cluster_key/databricks.yml @@ -11,8 +11,8 @@ targets: foo: name: job tasks: - - task_key: test - job_cluster_key: key + - task_key: test + job_cluster_key: key development: resources: jobs: @@ -23,5 +23,5 @@ targets: node_type_id: i3.xlarge num_workers: 1 tasks: - - task_key: test - job_cluster_key: key + - task_key: test + job_cluster_key: key diff --git a/bundle/tests/relative_path_with_includes/subfolder/include.yml b/bundle/tests/relative_path_with_includes/subfolder/include.yml index 597abe3bf1..87b7f4e99e 100644 --- a/bundle/tests/relative_path_with_includes/subfolder/include.yml +++ b/bundle/tests/relative_path_with_includes/subfolder/include.yml @@ -9,7 +9,6 @@ artifacts: type: whl path: ./artifact_b - resources: jobs: job_b: diff --git a/bundle/tests/run_as/legacy/databricks.yml b/bundle/tests/run_as/legacy/databricks.yml index e47224dbb9..86aa5d77be 100644 --- a/bundle/tests/run_as/legacy/databricks.yml +++ b/bundle/tests/run_as/legacy/databricks.yml @@ -50,7 +50,6 @@ resources: - notebook: path: ./dlt/nyc_taxi_loader - models: model_one: name: "skynet" diff --git a/bundle/tests/yaml_anchors_separate_block/databricks.yml b/bundle/tests/yaml_anchors_separate_block/databricks.yml index 447d5d0bb3..dc93523203 100644 --- a/bundle/tests/yaml_anchors_separate_block/databricks.yml +++ b/bundle/tests/yaml_anchors_separate_block/databricks.yml @@ -10,6 +10,6 @@ resources: jobs: my_job: tasks: - - task_key: yaml_anchors_separate_block + - task_key: yaml_anchors_separate_block tags: <<: *custom_tags diff --git a/cmd/account/log-delivery/log-delivery.go b/cmd/account/log-delivery/log-delivery.go index e2833263b2..036ea0c449 100755 --- a/cmd/account/log-delivery/log-delivery.go +++ b/cmd/account/log-delivery/log-delivery.go @@ -20,66 +20,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "log-delivery", - Short: `These APIs manage log delivery configurations for this account.`, - Long: `These APIs manage log delivery configurations for this account. The two - supported log types for this API are _billable usage logs_ and _audit logs_. - This feature is in Public Preview. This feature works with all account ID - types. - - Log delivery works with all account types. However, if your account is on the - E2 version of the platform or on a select custom plan that allows multiple - workspaces per account, you can optionally configure different storage - destinations for each workspace. Log delivery status is also provided to know - the latest status of log delivery attempts. The high-level flow of billable - usage delivery: - - 1. **Create storage**: In AWS, [create a new AWS S3 bucket] with a specific - bucket policy. Using Databricks APIs, call the Account API to create a - [storage configuration object](:method:Storage/Create) that uses the bucket - name. 2. **Create credentials**: In AWS, create the appropriate AWS IAM role. - For full details, including the required IAM role policies and trust - relationship, see [Billable usage log delivery]. Using Databricks APIs, call - the Account API to create a [credential configuration - object](:method:Credentials/Create) that uses the IAM role"s ARN. 3. **Create - log delivery configuration**: Using Databricks APIs, call the Account API to - [create a log delivery configuration](:method:LogDelivery/Create) that uses - the credential and storage configuration objects from previous steps. You can - specify if the logs should include all events of that log type in your account - (_Account level_ delivery) or only events for a specific set of workspaces - (_workspace level_ delivery). Account level log delivery applies to all - current and future workspaces plus account level logs, while workspace level - log delivery solely delivers logs related to the specified workspaces. You can - create multiple types of delivery configurations per account. - - For billable usage delivery: * For more information about billable usage logs, - see [Billable usage log delivery]. For the CSV schema, see the [Usage page]. * - The delivery location is //billable-usage/csv/, where - is the name of the optional delivery path prefix you set up during - log delivery configuration. Files are named - workspaceId=-usageMonth=.csv. * All billable usage logs - apply to specific workspaces (_workspace level_ logs). You can aggregate usage - for your entire account by creating an _account level_ delivery configuration - that delivers logs for all current and future workspaces in your account. * - The files are delivered daily by overwriting the month's CSV file for each - workspace. - - For audit log delivery: * For more information about about audit log delivery, - see [Audit log delivery], which includes information about the used JSON - schema. * The delivery location is - //workspaceId=/date=/auditlogs_.json. - Files may get overwritten with the same content multiple times to achieve - exactly-once delivery. * If the audit log delivery configuration included - specific workspace IDs, only _workspace-level_ audit logs for those workspaces - are delivered. If the log delivery configuration applies to the entire account - (_account level_ delivery configuration), the audit log delivery includes - workspace-level audit logs for all workspaces in the account as well as - account-level audit logs. See [Audit log delivery] for details. * Auditable - events are typically available in logs within 15 minutes. - - [Audit log delivery]: https://docs.databricks.com/administration-guide/account-settings/audit-logs.html - [Billable usage log delivery]: https://docs.databricks.com/administration-guide/account-settings/billable-usage-delivery.html - [Usage page]: https://docs.databricks.com/administration-guide/account-settings/usage.html - [create a new AWS S3 bucket]: https://docs.databricks.com/administration-guide/account-api/aws-storage.html`, + Short: `These APIs manage Log delivery configurations for this account.`, + Long: `These APIs manage Log delivery configurations for this account. Log delivery + configs enable you to configure the delivery of the specified type of logs to + your storage account.`, GroupID: "billing", Annotations: map[string]string{ "package": "billing", @@ -119,8 +63,6 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - // TODO: complex arg: log_delivery_configuration - cmd.Use = "create" cmd.Short = `Create a new log delivery configuration.` cmd.Long = `Create a new log delivery configuration. @@ -153,11 +95,6 @@ func newCreate() *cobra.Command { cmd.Annotations = make(map[string]string) - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(0) - return check(cmd, args) - } - cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -174,6 +111,8 @@ func newCreate() *cobra.Command { return err } } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := a.LogDelivery.Create(ctx, createReq) @@ -219,7 +158,7 @@ func newGet() *cobra.Command { specified by ID. Arguments: - LOG_DELIVERY_CONFIGURATION_ID: Databricks log delivery configuration ID` + LOG_DELIVERY_CONFIGURATION_ID: The log delivery configuration id of customer` cmd.Annotations = make(map[string]string) @@ -236,14 +175,14 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Log Delivery drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "Databricks log delivery configuration ID") + id, err := cmdio.Select(ctx, names, "The log delivery configuration id of customer") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have databricks log delivery configuration id") + return fmt.Errorf("expected to have the log delivery configuration id of customer") } getReq.LogDeliveryConfigurationId = args[0] @@ -282,9 +221,10 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().StringVar(&listReq.CredentialsId, "credentials-id", listReq.CredentialsId, `Filter by credential configuration ID.`) - cmd.Flags().Var(&listReq.Status, "status", `Filter by status ENABLED or DISABLED. Supported values: [DISABLED, ENABLED]`) - cmd.Flags().StringVar(&listReq.StorageConfigurationId, "storage-configuration-id", listReq.StorageConfigurationId, `Filter by storage configuration ID.`) + cmd.Flags().StringVar(&listReq.CredentialsId, "credentials-id", listReq.CredentialsId, `The Credentials id to filter the search results with.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token received from a previous get all budget configurations call.`) + cmd.Flags().Var(&listReq.Status, "status", `The log delivery status to filter the search results with. Supported values: [DISABLED, ENABLED]`) + cmd.Flags().StringVar(&listReq.StorageConfigurationId, "storage-configuration-id", listReq.StorageConfigurationId, `The Storage Configuration id to filter the search results with.`) cmd.Use = "list" cmd.Short = `Get all log delivery configurations.` @@ -350,7 +290,7 @@ func newPatchStatus() *cobra.Command { [Create log delivery](:method:LogDelivery/Create). Arguments: - LOG_DELIVERY_CONFIGURATION_ID: Databricks log delivery configuration ID + LOG_DELIVERY_CONFIGURATION_ID: The log delivery configuration id of customer STATUS: Status of log delivery configuration. Set to ENABLED (enabled) or DISABLED (disabled). Defaults to ENABLED. You can [enable or disable the configuration](#operation/patch-log-delivery-config-status) later. diff --git a/cmd/account/network-connectivity/network-connectivity.go b/cmd/account/network-connectivity/network-connectivity.go index 5b098ed01d..b7682f8780 100755 --- a/cmd/account/network-connectivity/network-connectivity.go +++ b/cmd/account/network-connectivity/network-connectivity.go @@ -46,7 +46,7 @@ func New() *cobra.Command { cmd.AddCommand(newGetPrivateEndpointRule()) cmd.AddCommand(newListNetworkConnectivityConfigurations()) cmd.AddCommand(newListPrivateEndpointRules()) - cmd.AddCommand(newUpdateNccAzurePrivateEndpointRulePublic()) + cmd.AddCommand(newUpdatePrivateEndpointRule()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -178,9 +178,12 @@ func newCreatePrivateEndpointRule() *cobra.Command { cmd.Flags().Var(&createPrivateEndpointRuleJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: domain_names - cmd.Flags().StringVar(&createPrivateEndpointRuleReq.PrivateEndpointRule.GroupId, "group-id", createPrivateEndpointRuleReq.PrivateEndpointRule.GroupId, `Only used by private endpoints to Azure first-party services.`) + cmd.Flags().StringVar(&createPrivateEndpointRuleReq.PrivateEndpointRule.EndpointService, "endpoint-service", createPrivateEndpointRuleReq.PrivateEndpointRule.EndpointService, `The full target AWS endpoint service name that connects to the destination resources of the private endpoint.`) + cmd.Flags().StringVar(&createPrivateEndpointRuleReq.PrivateEndpointRule.GroupId, "group-id", createPrivateEndpointRuleReq.PrivateEndpointRule.GroupId, `Not used by customer-managed private endpoint services.`) + cmd.Flags().StringVar(&createPrivateEndpointRuleReq.PrivateEndpointRule.ResourceId, "resource-id", createPrivateEndpointRuleReq.PrivateEndpointRule.ResourceId, `The Azure resource ID of the target resource.`) + // TODO: array: resource_names - cmd.Use = "create-private-endpoint-rule NETWORK_CONNECTIVITY_CONFIG_ID RESOURCE_ID" + cmd.Use = "create-private-endpoint-rule NETWORK_CONNECTIVITY_CONFIG_ID" cmd.Short = `Create a private endpoint rule.` cmd.Long = `Create a private endpoint rule. @@ -196,20 +199,12 @@ func newCreatePrivateEndpointRule() *cobra.Command { [serverless private link]: https://learn.microsoft.com/azure/databricks/security/network/serverless-network-security/serverless-private-link Arguments: - NETWORK_CONNECTIVITY_CONFIG_ID: Your Network Connectivity Configuration ID. - RESOURCE_ID: The Azure resource ID of the target resource.` + NETWORK_CONNECTIVITY_CONFIG_ID: Your Network Connectivity Configuration ID.` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("json") { - err := root.ExactArgs(1)(cmd, args) - if err != nil { - return fmt.Errorf("when --json flag is specified, provide only NETWORK_CONNECTIVITY_CONFIG_ID as positional arguments. Provide 'resource_id' in your JSON input") - } - return nil - } - check := root.ExactArgs(2) + check := root.ExactArgs(1) return check(cmd, args) } @@ -231,9 +226,6 @@ func newCreatePrivateEndpointRule() *cobra.Command { } } createPrivateEndpointRuleReq.NetworkConnectivityConfigId = args[0] - if !cmd.Flags().Changed("json") { - createPrivateEndpointRuleReq.PrivateEndpointRule.ResourceId = args[1] - } response, err := a.NetworkConnectivity.CreatePrivateEndpointRule(ctx, createPrivateEndpointRuleReq) if err != nil { @@ -604,28 +596,30 @@ func newListPrivateEndpointRules() *cobra.Command { return cmd } -// start update-ncc-azure-private-endpoint-rule-public command +// start update-private-endpoint-rule command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var updateNccAzurePrivateEndpointRulePublicOverrides []func( +var updatePrivateEndpointRuleOverrides []func( *cobra.Command, - *settings.UpdateNccAzurePrivateEndpointRulePublicRequest, + *settings.UpdateNccPrivateEndpointRuleRequest, ) -func newUpdateNccAzurePrivateEndpointRulePublic() *cobra.Command { +func newUpdatePrivateEndpointRule() *cobra.Command { cmd := &cobra.Command{} - var updateNccAzurePrivateEndpointRulePublicReq settings.UpdateNccAzurePrivateEndpointRulePublicRequest - updateNccAzurePrivateEndpointRulePublicReq.PrivateEndpointRule = settings.UpdatePrivateEndpointRule{} - var updateNccAzurePrivateEndpointRulePublicJson flags.JsonFlag + var updatePrivateEndpointRuleReq settings.UpdateNccPrivateEndpointRuleRequest + updatePrivateEndpointRuleReq.PrivateEndpointRule = settings.UpdatePrivateEndpointRule{} + var updatePrivateEndpointRuleJson flags.JsonFlag // TODO: short flags - cmd.Flags().Var(&updateNccAzurePrivateEndpointRulePublicJson, "json", `either inline JSON string or @path/to/file.json with request body`) + cmd.Flags().Var(&updatePrivateEndpointRuleJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: domain_names + cmd.Flags().BoolVar(&updatePrivateEndpointRuleReq.PrivateEndpointRule.Enabled, "enabled", updatePrivateEndpointRuleReq.PrivateEndpointRule.Enabled, `Only used by private endpoints towards an AWS S3 service.`) + // TODO: array: resource_names - cmd.Use = "update-ncc-azure-private-endpoint-rule-public NETWORK_CONNECTIVITY_CONFIG_ID PRIVATE_ENDPOINT_RULE_ID" + cmd.Use = "update-private-endpoint-rule NETWORK_CONNECTIVITY_CONFIG_ID PRIVATE_ENDPOINT_RULE_ID" cmd.Short = `Update a private endpoint rule.` cmd.Long = `Update a private endpoint rule. @@ -633,12 +627,10 @@ func newUpdateNccAzurePrivateEndpointRulePublic() *cobra.Command { customer-managed resources is allowed to be updated. Arguments: - NETWORK_CONNECTIVITY_CONFIG_ID: Your Network Connectivity Configuration ID. + NETWORK_CONNECTIVITY_CONFIG_ID: The ID of a network connectivity configuration, which is the parent + resource of this private endpoint rule object. PRIVATE_ENDPOINT_RULE_ID: Your private endpoint rule ID.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -652,7 +644,7 @@ func newUpdateNccAzurePrivateEndpointRulePublic() *cobra.Command { a := cmdctx.AccountClient(ctx) if cmd.Flags().Changed("json") { - diags := updateNccAzurePrivateEndpointRulePublicJson.Unmarshal(&updateNccAzurePrivateEndpointRulePublicReq.PrivateEndpointRule) + diags := updatePrivateEndpointRuleJson.Unmarshal(&updatePrivateEndpointRuleReq.PrivateEndpointRule) if diags.HasError() { return diags.Error() } @@ -663,10 +655,10 @@ func newUpdateNccAzurePrivateEndpointRulePublic() *cobra.Command { } } } - updateNccAzurePrivateEndpointRulePublicReq.NetworkConnectivityConfigId = args[0] - updateNccAzurePrivateEndpointRulePublicReq.PrivateEndpointRuleId = args[1] + updatePrivateEndpointRuleReq.NetworkConnectivityConfigId = args[0] + updatePrivateEndpointRuleReq.PrivateEndpointRuleId = args[1] - response, err := a.NetworkConnectivity.UpdateNccAzurePrivateEndpointRulePublic(ctx, updateNccAzurePrivateEndpointRulePublicReq) + response, err := a.NetworkConnectivity.UpdatePrivateEndpointRule(ctx, updatePrivateEndpointRuleReq) if err != nil { return err } @@ -678,8 +670,8 @@ func newUpdateNccAzurePrivateEndpointRulePublic() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range updateNccAzurePrivateEndpointRulePublicOverrides { - fn(cmd, &updateNccAzurePrivateEndpointRulePublicReq) + for _, fn := range updatePrivateEndpointRuleOverrides { + fn(cmd, &updatePrivateEndpointRuleReq) } return cmd diff --git a/cmd/account/network-policies/network-policies.go b/cmd/account/network-policies/network-policies.go index 00f5a72f71..c22c6576bb 100755 --- a/cmd/account/network-policies/network-policies.go +++ b/cmd/account/network-policies/network-policies.go @@ -30,10 +30,7 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "settings", }, - - // This service is being previewed; hide from help output. - Hidden: true, - RunE: root.ReportUnknownSubcommand, + RunE: root.ReportUnknownSubcommand, } // Add methods diff --git a/cmd/account/workspace-network-configuration/workspace-network-configuration.go b/cmd/account/workspace-network-configuration/workspace-network-configuration.go index c94ac5a96f..dee1c53af5 100755 --- a/cmd/account/workspace-network-configuration/workspace-network-configuration.go +++ b/cmd/account/workspace-network-configuration/workspace-network-configuration.go @@ -20,22 +20,19 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "workspace-network-configuration", - Short: `These APIs allow configuration of network settings for Databricks workspaces.`, - Long: `These APIs allow configuration of network settings for Databricks workspaces. - Each workspace is always associated with exactly one network policy that - controls which network destinations can be accessed from the Databricks - environment. By default, workspaces are associated with the 'default-policy' - network policy. You cannot create or delete a workspace's network - configuration, only update it to associate the workspace with a different - policy.`, + Short: `These APIs allow configuration of network settings for Databricks workspaces by selecting which network policy to associate with the workspace.`, + Long: `These APIs allow configuration of network settings for Databricks workspaces + by selecting which network policy to associate with the workspace. Each + workspace is always associated with exactly one network policy that controls + which network destinations can be accessed from the Databricks environment. By + default, workspaces are associated with the 'default-policy' network policy. + You cannot create or delete a workspace's network option, only update it to + associate the workspace with a different policy`, GroupID: "settings", Annotations: map[string]string{ "package": "settings", }, - - // This service is being previewed; hide from help output. - Hidden: true, - RunE: root.ReportUnknownSubcommand, + RunE: root.ReportUnknownSubcommand, } // Add methods @@ -67,12 +64,12 @@ func newGetWorkspaceNetworkOptionRpc() *cobra.Command { // TODO: short flags cmd.Use = "get-workspace-network-option-rpc WORKSPACE_ID" - cmd.Short = `Get workspace network configuration.` - cmd.Long = `Get workspace network configuration. + cmd.Short = `Get workspace network option.` + cmd.Long = `Get workspace network option. - Gets the network configuration for a workspace. Every workspace has exactly - one network policy binding, with 'default-policy' used if no explicit - assignment exists. + Gets the network option for a workspace. Every workspace has exactly one + network policy binding, with 'default-policy' used if no explicit assignment + exists. Arguments: WORKSPACE_ID: The workspace ID.` @@ -136,12 +133,12 @@ func newUpdateWorkspaceNetworkOptionRpc() *cobra.Command { cmd.Flags().Int64Var(&updateWorkspaceNetworkOptionRpcReq.WorkspaceNetworkOption.WorkspaceId, "workspace-id", updateWorkspaceNetworkOptionRpcReq.WorkspaceNetworkOption.WorkspaceId, `The workspace ID.`) cmd.Use = "update-workspace-network-option-rpc WORKSPACE_ID" - cmd.Short = `Update workspace network configuration.` - cmd.Long = `Update workspace network configuration. + cmd.Short = `Update workspace network option.` + cmd.Long = `Update workspace network option. - Updates the network configuration for a workspace. This operation associates - the workspace with the specified network policy. To revert to the default - policy, specify 'default-policy' as the network_policy_id. + Updates the network option for a workspace. This operation associates the + workspace with the specified network policy. To revert to the default policy, + specify 'default-policy' as the network_policy_id. Arguments: WORKSPACE_ID: The workspace ID.` diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 92cd2f1640..b7ad0b68fa 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" @@ -353,7 +354,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) diag.D diags = diags.Extend(bundle.ApplySeq(ctx, b, terraform.Interpolate(), terraform.Write(), - terraform.StatePull(), + statemgmt.StatePull(), terraform.Load(), )) if diags.HasError() { diff --git a/cmd/bundle/open.go b/cmd/bundle/open.go index 733758a8e8..5929d28819 100644 --- a/cmd/bundle/open.go +++ b/cmd/bundle/open.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" @@ -87,7 +88,7 @@ func newOpenCommand() *cobra.Command { if forcePull || noCache { diags = bundle.ApplySeq(ctx, b, - terraform.StatePull(), + statemgmt.StatePull(), terraform.Interpolate(), terraform.Write(), ) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index f9f5ac7abc..709cb1c87a 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run" "github.com/databricks/cli/bundle/run/output" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" @@ -151,7 +152,7 @@ Example usage: diags = diags.Extend(bundle.ApplySeq(ctx, b, terraform.Interpolate(), terraform.Write(), - terraform.StatePull(), + statemgmt.StatePull(), terraform.Load(terraform.ErrorOnEmptyState), )) if diags.HasError() { diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index d94c6d6eb5..9d0e11d848 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/render" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/flags" @@ -53,7 +54,7 @@ func newSummaryCommand() *cobra.Command { if forcePull || noCache { diags = bundle.ApplySeq(ctx, b, - terraform.StatePull(), + statemgmt.StatePull(), terraform.Interpolate(), terraform.Write(), ) diff --git a/cmd/cmd.go b/cmd/cmd.go index 7fe25281fc..465cf0d65d 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -2,8 +2,6 @@ package cmd import ( "context" - "os" - "path/filepath" "strings" "github.com/databricks/cli/cmd/account" @@ -27,19 +25,50 @@ const ( permissionsGroup = "permissions" ) -func New(ctx context.Context) *cobra.Command { - invokedAs := filepath.Base(os.Args[0]) - if invokedAs == "dlt" { - return dlt.NewRoot() +// filterGroups returns command groups that have at least one available (non-hidden) command. +// Empty groups or groups with only hidden commands are filtered out from the help output. +// Commands that belong to filtered groups will have their GroupID cleared. +func filterGroups(groups []cobra.Group, allCommands []*cobra.Command) []cobra.Group { + var filteredGroups []cobra.Group + + // Create a map to track which groups have available commands + groupHasAvailableCommands := make(map[string]bool) + + // Check each command to see if it belongs to a group and is available + for _, cmd := range allCommands { + if cmd.GroupID != "" && cmd.IsAvailableCommand() { + groupHasAvailableCommands[cmd.GroupID] = true + } } + // Collect groups that have available commands + validGroupIDs := make(map[string]bool) + for _, group := range groups { + if groupHasAvailableCommands[group.ID] { + filteredGroups = append(filteredGroups, group) + validGroupIDs[group.ID] = true + } + } + + // Clear GroupID for commands that belong to filtered groups + for _, cmd := range allCommands { + if cmd.GroupID != "" && !validGroupIDs[cmd.GroupID] { + cmd.GroupID = "" + } + } + + return filteredGroups +} + +func New(ctx context.Context) *cobra.Command { cli := root.New(ctx) // Add account subcommand. cli.AddCommand(account.New()) // Add workspace subcommands. - for _, cmd := range workspace.All() { + workspaceCommands := workspace.All() + for _, cmd := range workspaceCommands { // Built-in groups for the workspace commands. groups := []cobra.Group{ { @@ -68,12 +97,6 @@ func New(ctx context.Context) *cobra.Command { cli.AddCommand(cmd) } - // Add workspace command groups. - groups := workspace.Groups() - for i := range groups { - cli.AddGroup(&groups[i]) - } - // Add other subcommands. cli.AddCommand(api.New()) cli.AddCommand(auth.New()) @@ -84,7 +107,15 @@ func New(ctx context.Context) *cobra.Command { cli.AddCommand(sync.New()) cli.AddCommand(version.New()) cli.AddCommand(selftest.New()) - cli.AddCommand(dlt.New()) + cli.AddCommand(dlt.InstallDLT()) + + // Add workspace command groups, filtering out empty groups or groups with only hidden commands. + allGroups := workspace.Groups() + allCommands := cli.Commands() + filteredGroups := filterGroups(allGroups, allCommands) + for i := range filteredGroups { + cli.AddGroup(&filteredGroups[i]) + } return cli } diff --git a/cmd/dlt/__debug_bin60659176 b/cmd/dlt/__debug_bin60659176 new file mode 100755 index 0000000000..57caf9c5a0 Binary files /dev/null and b/cmd/dlt/__debug_bin60659176 differ diff --git a/cmd/dlt/dlt.go b/cmd/dlt/dlt.go index 34b7cf0ad4..3577d8924f 100644 --- a/cmd/dlt/dlt.go +++ b/cmd/dlt/dlt.go @@ -1,85 +1,115 @@ package dlt import ( - "context" "errors" + "encoding/json" "fmt" "os" - "os/exec" - "path/filepath" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/template" "github.com/spf13/cobra" ) -// InstallDLTSymlink creates a symlink named 'dlt' pointing to the real databricks binary. -func InstallDLTSymlink() error { - path, err := exec.LookPath("databricks") +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "dlt", + Short: "DLT CLI", + Long: "DLT CLI (stub, to be filled in)", + Run: func(cmd *cobra.Command, args []string) { + _ = cmd.Help() + }, + } + + cmd.AddCommand(NewInit()) + return cmd +} + +// createDefaultConfig creates a temporary config file with default values +func createDefaultConfig(projectName string) (string, error) { + config := map[string]any{} + + if projectName != "" { + config["project_name"] = projectName + } + + // Create JSON content + bytes, err := json.Marshal(config) if err != nil { - return errors.New("databricks CLI not found in PATH") + return "", fmt.Errorf("failed to marshal config: %w", err) } - realPath, err := filepath.EvalSymlinks(path) + + // Create temporary file + tmpFile, err := os.CreateTemp("", "dlt-config-*.json") if err != nil { - return fmt.Errorf("failed to resolve symlink: %w", err) + return "", fmt.Errorf("failed to create temporary config file: %w", err) } - dir := filepath.Dir(path) - dltPath := filepath.Join(dir, "dlt") - - // Check if DLT already exists - if fi, err := os.Lstat(dltPath); err == nil { - if fi.Mode()&os.ModeSymlink != 0 { - target, err := os.Readlink(dltPath) - if err == nil && target == realPath { - // DLT is already installed, so we can return success - cmdio.LogString(context.Background(), "dlt successfully installed") - return nil - } - } - return fmt.Errorf("cannot create symlink: %q already exists", dltPath) - } else if !os.IsNotExist(err) { - // Some other error occurred while checking - return fmt.Errorf("failed to check if %q exists: %w", dltPath, err) + // Write config to file + if _, err := tmpFile.Write(bytes); err != nil { + tmpFile.Close() + os.Remove(tmpFile.Name()) + return "", fmt.Errorf("failed to write config file: %w", err) } - if err := os.Symlink(realPath, dltPath); err != nil { - return fmt.Errorf("failed to create symlink: %w", err) + if err := tmpFile.Close(); err != nil { + os.Remove(tmpFile.Name()) + return "", fmt.Errorf("failed to close config file: %w", err) } - cmdio.LogString(context.Background(), "dlt successfully installed") - return nil + + return tmpFile.Name(), nil } -func New() *cobra.Command { + +func NewInit() *cobra.Command { return &cobra.Command{ - Use: "install-dlt", - Short: "Install DLT", - Hidden: true, - RunE: func(cmd *cobra.Command, args []string) error { - return InstallDLTSymlink() - }, + Use: "init [PROJECT_NAME]", + Short: "Initialize a new DLT pipeline project", + Long: "Initialize a new DLT pipeline project using the dlt template.", + Args: cobra.MaximumNArgs(1), + PreRunE: root.MustWorkspaceClient, + RunE: initDLTProject, } } -func NewRoot() *cobra.Command { - cmd := &cobra.Command{ - Use: "dlt", - Short: "DLT CLI", - Long: "DLT CLI (stub, to be filled in)", - Run: func(cmd *cobra.Command, args []string) { - _ = cmd.Help() - }, +func initDLTProject(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + // Extract project name from arguments if provided + var projectName string + if len(args) > 0 { + projectName = args[0] + cmdio.LogString(ctx, fmt.Sprintf("Project name (provided via command line): %s", projectName)) } + // Create a temporary config file with defaults + configFile, err := createDefaultConfig(projectName) + if err != nil { + return err + } + defer os.Remove(configFile) - // Add 'init' stub command (same description as bundle init) - initCmd := &cobra.Command{ - Use: "init", - Short: "Initialize a new DLT project in the current directory", - Long: "Initialize a new DLT project in the current directory. This is a stub for future implementation.", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("dlt init is not yet implemented. This will initialize a new DLT project in the future.") - }, + r := template.Resolver{ + TemplatePathOrUrl: "dlt", + ConfigFile: configFile, + OutputDir: ".", } - cmd.AddCommand(initCmd) - return cmd + tmpl, err := r.Resolve(ctx) + if errors.Is(err, template.ErrCustomSelected) { + cmdio.LogString(ctx, "Please specify a path or Git repository to use a custom template.") + cmdio.LogString(ctx, "See https://docs.databricks.com/en/dev-tools/bundles/templates.html to learn more about custom templates.") + return nil + } + if err != nil { + return fmt.Errorf("failed to resolve dlt template: %w", err) + } + defer tmpl.Reader.Cleanup(ctx) + + err = tmpl.Writer.Materialize(ctx, tmpl.Reader) + if err != nil { + return fmt.Errorf("failed to create DLT pipeline project: %w", err) + } + + return nil } diff --git a/cmd/dlt/install_dlt.go b/cmd/dlt/install_dlt.go new file mode 100644 index 0000000000..3a85e03202 --- /dev/null +++ b/cmd/dlt/install_dlt.go @@ -0,0 +1,63 @@ +package dlt + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/libs/cmdio" + "github.com/spf13/cobra" +) + +func InstallDLTSymlink(directory string) error { + path, err := os.Executable() + if err != nil { + return errors.New("databricks CLI executable not found") + } + realPath, err := filepath.EvalSymlinks(path) + if err != nil { + return fmt.Errorf("failed to resolve symlink: %w", err) + } + + dir := directory + if dir == "" { + dir = filepath.Dir(path) + } + dltPath := filepath.Join(dir, "dlt") + + if fi, err := os.Lstat(dltPath); err == nil { + if fi.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(dltPath) + if err == nil && target != realPath { + return fmt.Errorf("cannot install dlt CLI: %q already exists", dltPath) + } + if err != nil { + return err + } + } + } else if os.IsNotExist(err) { + if err := os.Symlink(realPath, dltPath); err != nil { + return fmt.Errorf("failed to install dlt CLI: %w", err) + } + } else { + return fmt.Errorf("failed to check if %q exists: %w", dltPath, err) + } + cmdio.LogString(context.Background(), fmt.Sprintf("dlt successfully installed to the directory %q", dir)) + return nil +} + +func InstallDLT() *cobra.Command { + var directory string + cmd := &cobra.Command{ + Use: "install-dlt", + Short: "Install DLT", + Hidden: true, + RunE: func(cmd *cobra.Command, args []string) error { + return InstallDLTSymlink(directory) + }, + } + cmd.Flags().StringVarP(&directory, "directory", "d", "", "Directory in which to install dlt CLI (defaults to databricks CLI's directory)") + return cmd +} diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml index b8a0e695e4..3918678d65 100644 --- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml +++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml @@ -1,4 +1,3 @@ ---- version: 1 name: blueprint description: Blueprint Project @@ -28,7 +27,7 @@ commands: description: second flag description - name: table description: something that renders a table - table_template: | + table_template: |- Key Value {{range .records}}{{.key}} {{.value}} {{end}} diff --git a/cmd/workspace/alerts-v2/alerts-v2.go b/cmd/workspace/alerts-v2/alerts-v2.go index 37db596a0e..1e1267fc4e 100755 --- a/cmd/workspace/alerts-v2/alerts-v2.go +++ b/cmd/workspace/alerts-v2/alerts-v2.go @@ -20,16 +20,13 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "alerts-v2", - Short: `TODO: Add description.`, - Long: `TODO: Add description`, + Short: `New version of SQL Alerts.`, + Long: `New version of SQL Alerts`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", }, - - // This service is being previewed; hide from help output. - Hidden: true, - RunE: root.ReportUnknownSubcommand, + RunE: root.ReportUnknownSubcommand, } // Add methods diff --git a/cmd/workspace/clean-room-assets/clean-room-assets.go b/cmd/workspace/clean-room-assets/clean-room-assets.go index e8c4b9cb3b..14242dd59b 100755 --- a/cmd/workspace/clean-room-assets/clean-room-assets.go +++ b/cmd/workspace/clean-room-assets/clean-room-assets.go @@ -149,7 +149,7 @@ func newDelete() *cobra.Command { // TODO: short flags - cmd.Use = "delete CLEAN_ROOM_NAME ASSET_TYPE ASSET_FULL_NAME" + cmd.Use = "delete CLEAN_ROOM_NAME ASSET_TYPE NAME" cmd.Short = `Delete an asset.` cmd.Long = `Delete an asset. @@ -159,7 +159,7 @@ func newDelete() *cobra.Command { CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: The type of the asset. Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] - ASSET_FULL_NAME: The fully qualified name of the asset, it is same as the name field in + NAME: The fully qualified name of the asset, it is same as the name field in CleanRoomAsset.` cmd.Annotations = make(map[string]string) @@ -179,7 +179,7 @@ func newDelete() *cobra.Command { if err != nil { return fmt.Errorf("invalid ASSET_TYPE: %s", args[1]) } - deleteReq.AssetFullName = args[2] + deleteReq.Name = args[2] err = w.CleanRoomAssets.Delete(ctx, deleteReq) if err != nil { @@ -216,7 +216,7 @@ func newGet() *cobra.Command { // TODO: short flags - cmd.Use = "get CLEAN_ROOM_NAME ASSET_TYPE ASSET_FULL_NAME" + cmd.Use = "get CLEAN_ROOM_NAME ASSET_TYPE NAME" cmd.Short = `Get an asset.` cmd.Long = `Get an asset. @@ -226,7 +226,7 @@ func newGet() *cobra.Command { CLEAN_ROOM_NAME: Name of the clean room. ASSET_TYPE: The type of the asset. Supported values: [FOREIGN_TABLE, NOTEBOOK_FILE, TABLE, VIEW, VOLUME] - ASSET_FULL_NAME: The fully qualified name of the asset, it is same as the name field in + NAME: The fully qualified name of the asset, it is same as the name field in CleanRoomAsset.` cmd.Annotations = make(map[string]string) @@ -246,7 +246,7 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("invalid ASSET_TYPE: %s", args[1]) } - getReq.AssetFullName = args[2] + getReq.Name = args[2] response, err := w.CleanRoomAssets.Get(ctx, getReq) if err != nil { diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index c496d588bb..545b9c486e 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -24,10 +24,11 @@ import ( credentials "github.com/databricks/cli/cmd/workspace/credentials" credentials_manager "github.com/databricks/cli/cmd/workspace/credentials-manager" current_user "github.com/databricks/cli/cmd/workspace/current-user" + custom_llms "github.com/databricks/cli/cmd/workspace/custom-llms" dashboard_widgets "github.com/databricks/cli/cmd/workspace/dashboard-widgets" dashboards "github.com/databricks/cli/cmd/workspace/dashboards" data_sources "github.com/databricks/cli/cmd/workspace/data-sources" - database_instances "github.com/databricks/cli/cmd/workspace/database-instances" + database "github.com/databricks/cli/cmd/workspace/database" experiments "github.com/databricks/cli/cmd/workspace/experiments" external_locations "github.com/databricks/cli/cmd/workspace/external-locations" forecasting "github.com/databricks/cli/cmd/workspace/forecasting" @@ -63,10 +64,10 @@ import ( provider_provider_analytics_dashboards "github.com/databricks/cli/cmd/workspace/provider-provider-analytics-dashboards" provider_providers "github.com/databricks/cli/cmd/workspace/provider-providers" providers "github.com/databricks/cli/cmd/workspace/providers" + quality_monitor_v2 "github.com/databricks/cli/cmd/workspace/quality-monitor-v2" quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors" queries "github.com/databricks/cli/cmd/workspace/queries" queries_legacy "github.com/databricks/cli/cmd/workspace/queries-legacy" - query_execution "github.com/databricks/cli/cmd/workspace/query-execution" query_history "github.com/databricks/cli/cmd/workspace/query-history" query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations" query_visualizations_legacy "github.com/databricks/cli/cmd/workspace/query-visualizations-legacy" @@ -125,10 +126,11 @@ func All() []*cobra.Command { out = append(out, credentials.New()) out = append(out, credentials_manager.New()) out = append(out, current_user.New()) + out = append(out, custom_llms.New()) out = append(out, dashboard_widgets.New()) out = append(out, dashboards.New()) out = append(out, data_sources.New()) - out = append(out, database_instances.New()) + out = append(out, database.New()) out = append(out, experiments.New()) out = append(out, external_locations.New()) out = append(out, functions.New()) @@ -163,10 +165,10 @@ func All() []*cobra.Command { out = append(out, provider_provider_analytics_dashboards.New()) out = append(out, provider_providers.New()) out = append(out, providers.New()) + out = append(out, quality_monitor_v2.New()) out = append(out, quality_monitors.New()) out = append(out, queries.New()) out = append(out, queries_legacy.New()) - out = append(out, query_execution.New()) out = append(out, query_history.New()) out = append(out, query_visualizations.New()) out = append(out, query_visualizations_legacy.New()) diff --git a/cmd/workspace/custom-llms/custom-llms.go b/cmd/workspace/custom-llms/custom-llms.go new file mode 100755 index 0000000000..34ad043881 --- /dev/null +++ b/cmd/workspace/custom-llms/custom-llms.go @@ -0,0 +1,287 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package custom_llms + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/aibuilder" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "custom-llms", + Short: `The Custom LLMs service manages state and powers the UI for the Custom LLM product.`, + Long: `The Custom LLMs service manages state and powers the UI for the Custom LLM + product.`, + GroupID: "aibuilder", + Annotations: map[string]string{ + "package": "aibuilder", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + RunE: root.ReportUnknownSubcommand, + } + + // Add methods + cmd.AddCommand(newCancel()) + cmd.AddCommand(newCreate()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start cancel command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cancelOverrides []func( + *cobra.Command, + *aibuilder.CancelCustomLlmOptimizationRunRequest, +) + +func newCancel() *cobra.Command { + cmd := &cobra.Command{} + + var cancelReq aibuilder.CancelCustomLlmOptimizationRunRequest + + // TODO: short flags + + cmd.Use = "cancel ID" + cmd.Short = `Cancel a Custom LLM Optimization Run.` + cmd.Long = `Cancel a Custom LLM Optimization Run.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + cancelReq.Id = args[0] + + err = w.CustomLlms.Cancel(ctx, cancelReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range cancelOverrides { + fn(cmd, &cancelReq) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *aibuilder.StartCustomLlmOptimizationRunRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq aibuilder.StartCustomLlmOptimizationRunRequest + + // TODO: short flags + + cmd.Use = "create ID" + cmd.Short = `Start a Custom LLM Optimization Run.` + cmd.Long = `Start a Custom LLM Optimization Run. + + Arguments: + ID: The Id of the tile.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + createReq.Id = args[0] + + response, err := w.CustomLlms.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *aibuilder.GetCustomLlmRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq aibuilder.GetCustomLlmRequest + + // TODO: short flags + + cmd.Use = "get ID" + cmd.Short = `Get a Custom LLM.` + cmd.Long = `Get a Custom LLM. + + Arguments: + ID: The id of the custom llm` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getReq.Id = args[0] + + response, err := w.CustomLlms.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *aibuilder.UpdateCustomLlmRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq aibuilder.UpdateCustomLlmRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update ID" + cmd.Short = `Update a Custom LLM.` + cmd.Long = `Update a Custom LLM. + + Arguments: + ID: The id of the custom llm` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + updateReq.Id = args[0] + + response, err := w.CustomLlms.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service CustomLlms diff --git a/cmd/workspace/dashboard-email-subscriptions/dashboard-email-subscriptions.go b/cmd/workspace/dashboard-email-subscriptions/dashboard-email-subscriptions.go new file mode 100755 index 0000000000..0da11badd3 --- /dev/null +++ b/cmd/workspace/dashboard-email-subscriptions/dashboard-email-subscriptions.go @@ -0,0 +1,218 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package dashboard_email_subscriptions + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "dashboard-email-subscriptions", + Short: `Controls whether schedules or workload tasks for refreshing AI/BI Dashboards in the workspace can send subscription emails containing PDFs and/or images of the dashboard.`, + Long: `Controls whether schedules or workload tasks for refreshing AI/BI Dashboards + in the workspace can send subscription emails containing PDFs and/or images of + the dashboard. By default, this setting is enabled (set to true)`, + RunE: root.ReportUnknownSubcommand, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteDashboardEmailSubscriptionsRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteDashboardEmailSubscriptionsRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete the Dashboard Email Subscriptions setting.` + cmd.Long = `Delete the Dashboard Email Subscriptions setting. + + Reverts the Dashboard Email Subscriptions setting to its default value.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response, err := w.Settings.DashboardEmailSubscriptions().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetDashboardEmailSubscriptionsRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetDashboardEmailSubscriptionsRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Get the Dashboard Email Subscriptions setting.` + cmd.Long = `Get the Dashboard Email Subscriptions setting. + + Gets the Dashboard Email Subscriptions setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response, err := w.Settings.DashboardEmailSubscriptions().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateDashboardEmailSubscriptionsRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateDashboardEmailSubscriptionsRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update the Dashboard Email Subscriptions setting.` + cmd.Long = `Update the Dashboard Email Subscriptions setting. + + Updates the Dashboard Email Subscriptions setting.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.Settings.DashboardEmailSubscriptions().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service DashboardEmailSubscriptions diff --git a/cmd/workspace/database-instances/database-instances.go b/cmd/workspace/database/database.go similarity index 67% rename from cmd/workspace/database-instances/database-instances.go rename to cmd/workspace/database/database.go index 64fe1d7f82..f955d5953f 100755 --- a/cmd/workspace/database-instances/database-instances.go +++ b/cmd/workspace/database/database.go @@ -1,6 +1,6 @@ // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. -package database_instances +package database import ( "fmt" @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" - "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/database" "github.com/spf13/cobra" ) @@ -19,12 +19,12 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "database-instances", + Use: "database", Short: `Database Instances provide access to a database via REST API or direct SQL.`, Long: `Database Instances provide access to a database via REST API or direct SQL.`, - GroupID: "catalog", + GroupID: "database", Annotations: map[string]string{ - "package": "catalog", + "package": "database", }, // This service is being previewed; hide from help output. @@ -35,13 +35,17 @@ func New() *cobra.Command { // Add methods cmd.AddCommand(newCreateDatabaseCatalog()) cmd.AddCommand(newCreateDatabaseInstance()) + cmd.AddCommand(newCreateDatabaseTable()) cmd.AddCommand(newCreateSyncedDatabaseTable()) cmd.AddCommand(newDeleteDatabaseCatalog()) cmd.AddCommand(newDeleteDatabaseInstance()) + cmd.AddCommand(newDeleteDatabaseTable()) cmd.AddCommand(newDeleteSyncedDatabaseTable()) cmd.AddCommand(newFindDatabaseInstanceByUid()) + cmd.AddCommand(newGenerateDatabaseCredential()) cmd.AddCommand(newGetDatabaseCatalog()) cmd.AddCommand(newGetDatabaseInstance()) + cmd.AddCommand(newGetDatabaseTable()) cmd.AddCommand(newGetSyncedDatabaseTable()) cmd.AddCommand(newListDatabaseInstances()) cmd.AddCommand(newUpdateDatabaseInstance()) @@ -60,14 +64,14 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createDatabaseCatalogOverrides []func( *cobra.Command, - *catalog.CreateDatabaseCatalogRequest, + *database.CreateDatabaseCatalogRequest, ) func newCreateDatabaseCatalog() *cobra.Command { cmd := &cobra.Command{} - var createDatabaseCatalogReq catalog.CreateDatabaseCatalogRequest - createDatabaseCatalogReq.Catalog = catalog.DatabaseCatalog{} + var createDatabaseCatalogReq database.CreateDatabaseCatalogRequest + createDatabaseCatalogReq.Catalog = database.DatabaseCatalog{} var createDatabaseCatalogJson flags.JsonFlag // TODO: short flags @@ -125,7 +129,7 @@ func newCreateDatabaseCatalog() *cobra.Command { createDatabaseCatalogReq.Catalog.DatabaseName = args[2] } - response, err := w.DatabaseInstances.CreateDatabaseCatalog(ctx, createDatabaseCatalogReq) + response, err := w.Database.CreateDatabaseCatalog(ctx, createDatabaseCatalogReq) if err != nil { return err } @@ -150,21 +154,19 @@ func newCreateDatabaseCatalog() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createDatabaseInstanceOverrides []func( *cobra.Command, - *catalog.CreateDatabaseInstanceRequest, + *database.CreateDatabaseInstanceRequest, ) func newCreateDatabaseInstance() *cobra.Command { cmd := &cobra.Command{} - var createDatabaseInstanceReq catalog.CreateDatabaseInstanceRequest - createDatabaseInstanceReq.DatabaseInstance = catalog.DatabaseInstance{} + var createDatabaseInstanceReq database.CreateDatabaseInstanceRequest + createDatabaseInstanceReq.DatabaseInstance = database.DatabaseInstance{} var createDatabaseInstanceJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createDatabaseInstanceJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&createDatabaseInstanceReq.DatabaseInstance.AdminPassword, "admin-password", createDatabaseInstanceReq.DatabaseInstance.AdminPassword, `Password for admin user to create.`) - cmd.Flags().StringVar(&createDatabaseInstanceReq.DatabaseInstance.AdminRolename, "admin-rolename", createDatabaseInstanceReq.DatabaseInstance.AdminRolename, `Name of the admin role for the instance.`) cmd.Flags().StringVar(&createDatabaseInstanceReq.DatabaseInstance.Capacity, "capacity", createDatabaseInstanceReq.DatabaseInstance.Capacity, `The sku of the instance.`) cmd.Flags().BoolVar(&createDatabaseInstanceReq.DatabaseInstance.Stopped, "stopped", createDatabaseInstanceReq.DatabaseInstance.Stopped, `Whether the instance is stopped.`) @@ -210,7 +212,7 @@ func newCreateDatabaseInstance() *cobra.Command { createDatabaseInstanceReq.DatabaseInstance.Name = args[0] } - response, err := w.DatabaseInstances.CreateDatabaseInstance(ctx, createDatabaseInstanceReq) + response, err := w.Database.CreateDatabaseInstance(ctx, createDatabaseInstanceReq) if err != nil { return err } @@ -229,20 +231,103 @@ func newCreateDatabaseInstance() *cobra.Command { return cmd } +// start create-database-table command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createDatabaseTableOverrides []func( + *cobra.Command, + *database.CreateDatabaseTableRequest, +) + +func newCreateDatabaseTable() *cobra.Command { + cmd := &cobra.Command{} + + var createDatabaseTableReq database.CreateDatabaseTableRequest + createDatabaseTableReq.Table = database.DatabaseTable{} + var createDatabaseTableJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createDatabaseTableJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createDatabaseTableReq.Table.DatabaseInstanceName, "database-instance-name", createDatabaseTableReq.Table.DatabaseInstanceName, `Name of the target database instance.`) + cmd.Flags().StringVar(&createDatabaseTableReq.Table.LogicalDatabaseName, "logical-database-name", createDatabaseTableReq.Table.LogicalDatabaseName, `Target Postgres database object (logical database) name for this table.`) + + cmd.Use = "create-database-table NAME" + cmd.Short = `Create a Database Table.` + cmd.Long = `Create a Database Table. + + Arguments: + NAME: Full three-part (catalog, schema, table) name of the table.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createDatabaseTableJson.Unmarshal(&createDatabaseTableReq.Table) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + if !cmd.Flags().Changed("json") { + createDatabaseTableReq.Table.Name = args[0] + } + + response, err := w.Database.CreateDatabaseTable(ctx, createDatabaseTableReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createDatabaseTableOverrides { + fn(cmd, &createDatabaseTableReq) + } + + return cmd +} + // start create-synced-database-table command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. var createSyncedDatabaseTableOverrides []func( *cobra.Command, - *catalog.CreateSyncedDatabaseTableRequest, + *database.CreateSyncedDatabaseTableRequest, ) func newCreateSyncedDatabaseTable() *cobra.Command { cmd := &cobra.Command{} - var createSyncedDatabaseTableReq catalog.CreateSyncedDatabaseTableRequest - createSyncedDatabaseTableReq.SyncedTable = catalog.SyncedDatabaseTable{} + var createSyncedDatabaseTableReq database.CreateSyncedDatabaseTableRequest + createSyncedDatabaseTableReq.SyncedTable = database.SyncedDatabaseTable{} var createSyncedDatabaseTableJson flags.JsonFlag // TODO: short flags @@ -295,7 +380,7 @@ func newCreateSyncedDatabaseTable() *cobra.Command { createSyncedDatabaseTableReq.SyncedTable.Name = args[0] } - response, err := w.DatabaseInstances.CreateSyncedDatabaseTable(ctx, createSyncedDatabaseTableReq) + response, err := w.Database.CreateSyncedDatabaseTable(ctx, createSyncedDatabaseTableReq) if err != nil { return err } @@ -320,13 +405,13 @@ func newCreateSyncedDatabaseTable() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteDatabaseCatalogOverrides []func( *cobra.Command, - *catalog.DeleteDatabaseCatalogRequest, + *database.DeleteDatabaseCatalogRequest, ) func newDeleteDatabaseCatalog() *cobra.Command { cmd := &cobra.Command{} - var deleteDatabaseCatalogReq catalog.DeleteDatabaseCatalogRequest + var deleteDatabaseCatalogReq database.DeleteDatabaseCatalogRequest // TODO: short flags @@ -348,7 +433,7 @@ func newDeleteDatabaseCatalog() *cobra.Command { deleteDatabaseCatalogReq.Name = args[0] - err = w.DatabaseInstances.DeleteDatabaseCatalog(ctx, deleteDatabaseCatalogReq) + err = w.Database.DeleteDatabaseCatalog(ctx, deleteDatabaseCatalogReq) if err != nil { return err } @@ -373,13 +458,13 @@ func newDeleteDatabaseCatalog() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteDatabaseInstanceOverrides []func( *cobra.Command, - *catalog.DeleteDatabaseInstanceRequest, + *database.DeleteDatabaseInstanceRequest, ) func newDeleteDatabaseInstance() *cobra.Command { cmd := &cobra.Command{} - var deleteDatabaseInstanceReq catalog.DeleteDatabaseInstanceRequest + var deleteDatabaseInstanceReq database.DeleteDatabaseInstanceRequest // TODO: short flags @@ -407,7 +492,7 @@ func newDeleteDatabaseInstance() *cobra.Command { deleteDatabaseInstanceReq.Name = args[0] - err = w.DatabaseInstances.DeleteDatabaseInstance(ctx, deleteDatabaseInstanceReq) + err = w.Database.DeleteDatabaseInstance(ctx, deleteDatabaseInstanceReq) if err != nil { return err } @@ -426,19 +511,72 @@ func newDeleteDatabaseInstance() *cobra.Command { return cmd } +// start delete-database-table command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteDatabaseTableOverrides []func( + *cobra.Command, + *database.DeleteDatabaseTableRequest, +) + +func newDeleteDatabaseTable() *cobra.Command { + cmd := &cobra.Command{} + + var deleteDatabaseTableReq database.DeleteDatabaseTableRequest + + // TODO: short flags + + cmd.Use = "delete-database-table NAME" + cmd.Short = `Delete a Database Table.` + cmd.Long = `Delete a Database Table.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + deleteDatabaseTableReq.Name = args[0] + + err = w.Database.DeleteDatabaseTable(ctx, deleteDatabaseTableReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteDatabaseTableOverrides { + fn(cmd, &deleteDatabaseTableReq) + } + + return cmd +} + // start delete-synced-database-table command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. var deleteSyncedDatabaseTableOverrides []func( *cobra.Command, - *catalog.DeleteSyncedDatabaseTableRequest, + *database.DeleteSyncedDatabaseTableRequest, ) func newDeleteSyncedDatabaseTable() *cobra.Command { cmd := &cobra.Command{} - var deleteSyncedDatabaseTableReq catalog.DeleteSyncedDatabaseTableRequest + var deleteSyncedDatabaseTableReq database.DeleteSyncedDatabaseTableRequest // TODO: short flags @@ -460,7 +598,7 @@ func newDeleteSyncedDatabaseTable() *cobra.Command { deleteSyncedDatabaseTableReq.Name = args[0] - err = w.DatabaseInstances.DeleteSyncedDatabaseTable(ctx, deleteSyncedDatabaseTableReq) + err = w.Database.DeleteSyncedDatabaseTable(ctx, deleteSyncedDatabaseTableReq) if err != nil { return err } @@ -485,13 +623,13 @@ func newDeleteSyncedDatabaseTable() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var findDatabaseInstanceByUidOverrides []func( *cobra.Command, - *catalog.FindDatabaseInstanceByUidRequest, + *database.FindDatabaseInstanceByUidRequest, ) func newFindDatabaseInstanceByUid() *cobra.Command { cmd := &cobra.Command{} - var findDatabaseInstanceByUidReq catalog.FindDatabaseInstanceByUidRequest + var findDatabaseInstanceByUidReq database.FindDatabaseInstanceByUidRequest // TODO: short flags @@ -513,7 +651,7 @@ func newFindDatabaseInstanceByUid() *cobra.Command { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - response, err := w.DatabaseInstances.FindDatabaseInstanceByUid(ctx, findDatabaseInstanceByUidReq) + response, err := w.Database.FindDatabaseInstanceByUid(ctx, findDatabaseInstanceByUidReq) if err != nil { return err } @@ -532,19 +670,88 @@ func newFindDatabaseInstanceByUid() *cobra.Command { return cmd } +// start generate-database-credential command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var generateDatabaseCredentialOverrides []func( + *cobra.Command, + *database.GenerateDatabaseCredentialRequest, +) + +func newGenerateDatabaseCredential() *cobra.Command { + cmd := &cobra.Command{} + + var generateDatabaseCredentialReq database.GenerateDatabaseCredentialRequest + var generateDatabaseCredentialJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&generateDatabaseCredentialJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: instance_names + cmd.Flags().StringVar(&generateDatabaseCredentialReq.RequestId, "request-id", generateDatabaseCredentialReq.RequestId, ``) + + cmd.Use = "generate-database-credential" + cmd.Short = `Generates a credential that can be used to access database instances.` + cmd.Long = `Generates a credential that can be used to access database instances.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := generateDatabaseCredentialJson.Unmarshal(&generateDatabaseCredentialReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + + response, err := w.Database.GenerateDatabaseCredential(ctx, generateDatabaseCredentialReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range generateDatabaseCredentialOverrides { + fn(cmd, &generateDatabaseCredentialReq) + } + + return cmd +} + // start get-database-catalog command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. var getDatabaseCatalogOverrides []func( *cobra.Command, - *catalog.GetDatabaseCatalogRequest, + *database.GetDatabaseCatalogRequest, ) func newGetDatabaseCatalog() *cobra.Command { cmd := &cobra.Command{} - var getDatabaseCatalogReq catalog.GetDatabaseCatalogRequest + var getDatabaseCatalogReq database.GetDatabaseCatalogRequest // TODO: short flags @@ -566,7 +773,7 @@ func newGetDatabaseCatalog() *cobra.Command { getDatabaseCatalogReq.Name = args[0] - response, err := w.DatabaseInstances.GetDatabaseCatalog(ctx, getDatabaseCatalogReq) + response, err := w.Database.GetDatabaseCatalog(ctx, getDatabaseCatalogReq) if err != nil { return err } @@ -591,13 +798,13 @@ func newGetDatabaseCatalog() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getDatabaseInstanceOverrides []func( *cobra.Command, - *catalog.GetDatabaseInstanceRequest, + *database.GetDatabaseInstanceRequest, ) func newGetDatabaseInstance() *cobra.Command { cmd := &cobra.Command{} - var getDatabaseInstanceReq catalog.GetDatabaseInstanceRequest + var getDatabaseInstanceReq database.GetDatabaseInstanceRequest // TODO: short flags @@ -622,7 +829,7 @@ func newGetDatabaseInstance() *cobra.Command { getDatabaseInstanceReq.Name = args[0] - response, err := w.DatabaseInstances.GetDatabaseInstance(ctx, getDatabaseInstanceReq) + response, err := w.Database.GetDatabaseInstance(ctx, getDatabaseInstanceReq) if err != nil { return err } @@ -641,19 +848,72 @@ func newGetDatabaseInstance() *cobra.Command { return cmd } +// start get-database-table command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getDatabaseTableOverrides []func( + *cobra.Command, + *database.GetDatabaseTableRequest, +) + +func newGetDatabaseTable() *cobra.Command { + cmd := &cobra.Command{} + + var getDatabaseTableReq database.GetDatabaseTableRequest + + // TODO: short flags + + cmd.Use = "get-database-table NAME" + cmd.Short = `Get a Database Table.` + cmd.Long = `Get a Database Table.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getDatabaseTableReq.Name = args[0] + + response, err := w.Database.GetDatabaseTable(ctx, getDatabaseTableReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getDatabaseTableOverrides { + fn(cmd, &getDatabaseTableReq) + } + + return cmd +} + // start get-synced-database-table command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. var getSyncedDatabaseTableOverrides []func( *cobra.Command, - *catalog.GetSyncedDatabaseTableRequest, + *database.GetSyncedDatabaseTableRequest, ) func newGetSyncedDatabaseTable() *cobra.Command { cmd := &cobra.Command{} - var getSyncedDatabaseTableReq catalog.GetSyncedDatabaseTableRequest + var getSyncedDatabaseTableReq database.GetSyncedDatabaseTableRequest // TODO: short flags @@ -675,7 +935,7 @@ func newGetSyncedDatabaseTable() *cobra.Command { getSyncedDatabaseTableReq.Name = args[0] - response, err := w.DatabaseInstances.GetSyncedDatabaseTable(ctx, getSyncedDatabaseTableReq) + response, err := w.Database.GetSyncedDatabaseTable(ctx, getSyncedDatabaseTableReq) if err != nil { return err } @@ -700,13 +960,13 @@ func newGetSyncedDatabaseTable() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listDatabaseInstancesOverrides []func( *cobra.Command, - *catalog.ListDatabaseInstancesRequest, + *database.ListDatabaseInstancesRequest, ) func newListDatabaseInstances() *cobra.Command { cmd := &cobra.Command{} - var listDatabaseInstancesReq catalog.ListDatabaseInstancesRequest + var listDatabaseInstancesReq database.ListDatabaseInstancesRequest // TODO: short flags @@ -729,7 +989,7 @@ func newListDatabaseInstances() *cobra.Command { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - response := w.DatabaseInstances.ListDatabaseInstances(ctx, listDatabaseInstancesReq) + response := w.Database.ListDatabaseInstances(ctx, listDatabaseInstancesReq) return cmdio.RenderIterator(ctx, response) } @@ -751,21 +1011,19 @@ func newListDatabaseInstances() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateDatabaseInstanceOverrides []func( *cobra.Command, - *catalog.UpdateDatabaseInstanceRequest, + *database.UpdateDatabaseInstanceRequest, ) func newUpdateDatabaseInstance() *cobra.Command { cmd := &cobra.Command{} - var updateDatabaseInstanceReq catalog.UpdateDatabaseInstanceRequest - updateDatabaseInstanceReq.DatabaseInstance = catalog.DatabaseInstance{} + var updateDatabaseInstanceReq database.UpdateDatabaseInstanceRequest + updateDatabaseInstanceReq.DatabaseInstance = database.DatabaseInstance{} var updateDatabaseInstanceJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateDatabaseInstanceJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateDatabaseInstanceReq.DatabaseInstance.AdminPassword, "admin-password", updateDatabaseInstanceReq.DatabaseInstance.AdminPassword, `Password for admin user to create.`) - cmd.Flags().StringVar(&updateDatabaseInstanceReq.DatabaseInstance.AdminRolename, "admin-rolename", updateDatabaseInstanceReq.DatabaseInstance.AdminRolename, `Name of the admin role for the instance.`) cmd.Flags().StringVar(&updateDatabaseInstanceReq.DatabaseInstance.Capacity, "capacity", updateDatabaseInstanceReq.DatabaseInstance.Capacity, `The sku of the instance.`) cmd.Flags().BoolVar(&updateDatabaseInstanceReq.DatabaseInstance.Stopped, "stopped", updateDatabaseInstanceReq.DatabaseInstance.Stopped, `Whether the instance is stopped.`) @@ -802,7 +1060,7 @@ func newUpdateDatabaseInstance() *cobra.Command { } updateDatabaseInstanceReq.Name = args[0] - response, err := w.DatabaseInstances.UpdateDatabaseInstance(ctx, updateDatabaseInstanceReq) + response, err := w.Database.UpdateDatabaseInstance(ctx, updateDatabaseInstanceReq) if err != nil { return err } @@ -821,4 +1079,4 @@ func newUpdateDatabaseInstance() *cobra.Command { return cmd } -// end service DatabaseInstances +// end service Database diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index d4a4738d07..1f94baf770 100755 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -49,8 +49,6 @@ func New() *cobra.Command { cmd.AddCommand(newDeleteTag()) cmd.AddCommand(newFinalizeLoggedModel()) cmd.AddCommand(newGetByName()) - cmd.AddCommand(newGetCredentialsForTraceDataDownload()) - cmd.AddCommand(newGetCredentialsForTraceDataUpload()) cmd.AddCommand(newGetExperiment()) cmd.AddCommand(newGetHistory()) cmd.AddCommand(newGetLoggedModel()) @@ -59,7 +57,6 @@ func New() *cobra.Command { cmd.AddCommand(newGetRun()) cmd.AddCommand(newListArtifacts()) cmd.AddCommand(newListExperiments()) - cmd.AddCommand(newListLoggedModelArtifacts()) cmd.AddCommand(newLogBatch()) cmd.AddCommand(newLogInputs()) cmd.AddCommand(newLogLoggedModelParams()) @@ -209,9 +206,6 @@ func newCreateLoggedModel() *cobra.Command { Arguments: EXPERIMENT_ID: The ID of the experiment that owns the model.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -449,9 +443,6 @@ func newDeleteLoggedModel() *cobra.Command { Arguments: MODEL_ID: The ID of the logged model to delete.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -509,9 +500,6 @@ func newDeleteLoggedModelTag() *cobra.Command { MODEL_ID: The ID of the logged model to delete the tag from. TAG_KEY: The tag key.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -833,12 +821,9 @@ func newFinalizeLoggedModel() *cobra.Command { MODEL_ID: The ID of the logged model to finalize. STATUS: Whether or not the model is ready for use. "LOGGED_MODEL_UPLOAD_FAILED" indicates that something went wrong when - logging the model weights / agent code). + logging the model weights / agent code. Supported values: [LOGGED_MODEL_PENDING, LOGGED_MODEL_READY, LOGGED_MODEL_UPLOAD_FAILED]` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -963,124 +948,6 @@ func newGetByName() *cobra.Command { return cmd } -// start get-credentials-for-trace-data-download command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var getCredentialsForTraceDataDownloadOverrides []func( - *cobra.Command, - *ml.GetCredentialsForTraceDataDownloadRequest, -) - -func newGetCredentialsForTraceDataDownload() *cobra.Command { - cmd := &cobra.Command{} - - var getCredentialsForTraceDataDownloadReq ml.GetCredentialsForTraceDataDownloadRequest - - // TODO: short flags - - cmd.Use = "get-credentials-for-trace-data-download REQUEST_ID" - cmd.Short = `Get credentials to download trace data.` - cmd.Long = `Get credentials to download trace data. - - Arguments: - REQUEST_ID: The ID of the trace to fetch artifact download credentials for.` - - // This command is being previewed; hide from help output. - cmd.Hidden = true - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - getCredentialsForTraceDataDownloadReq.RequestId = args[0] - - response, err := w.Experiments.GetCredentialsForTraceDataDownload(ctx, getCredentialsForTraceDataDownloadReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range getCredentialsForTraceDataDownloadOverrides { - fn(cmd, &getCredentialsForTraceDataDownloadReq) - } - - return cmd -} - -// start get-credentials-for-trace-data-upload command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var getCredentialsForTraceDataUploadOverrides []func( - *cobra.Command, - *ml.GetCredentialsForTraceDataUploadRequest, -) - -func newGetCredentialsForTraceDataUpload() *cobra.Command { - cmd := &cobra.Command{} - - var getCredentialsForTraceDataUploadReq ml.GetCredentialsForTraceDataUploadRequest - - // TODO: short flags - - cmd.Use = "get-credentials-for-trace-data-upload REQUEST_ID" - cmd.Short = `Get credentials to upload trace data.` - cmd.Long = `Get credentials to upload trace data. - - Arguments: - REQUEST_ID: The ID of the trace to fetch artifact upload credentials for.` - - // This command is being previewed; hide from help output. - cmd.Hidden = true - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - getCredentialsForTraceDataUploadReq.RequestId = args[0] - - response, err := w.Experiments.GetCredentialsForTraceDataUpload(ctx, getCredentialsForTraceDataUploadReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range getCredentialsForTraceDataUploadOverrides { - fn(cmd, &getCredentialsForTraceDataUploadReq) - } - - return cmd -} - // start get-experiment command // Slice with functions to override default command behavior. @@ -1222,9 +1089,6 @@ func newGetLoggedModel() *cobra.Command { Arguments: MODEL_ID: The ID of the logged model to retrieve.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -1554,72 +1418,6 @@ func newListExperiments() *cobra.Command { return cmd } -// start list-logged-model-artifacts command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var listLoggedModelArtifactsOverrides []func( - *cobra.Command, - *ml.ListLoggedModelArtifactsRequest, -) - -func newListLoggedModelArtifacts() *cobra.Command { - cmd := &cobra.Command{} - - var listLoggedModelArtifactsReq ml.ListLoggedModelArtifactsRequest - - // TODO: short flags - - cmd.Flags().StringVar(&listLoggedModelArtifactsReq.ArtifactDirectoryPath, "artifact-directory-path", listLoggedModelArtifactsReq.ArtifactDirectoryPath, `Filter artifacts matching this path (a relative path from the root artifact directory).`) - cmd.Flags().StringVar(&listLoggedModelArtifactsReq.PageToken, "page-token", listLoggedModelArtifactsReq.PageToken, `Token indicating the page of artifact results to fetch.`) - - cmd.Use = "list-logged-model-artifacts MODEL_ID" - cmd.Short = `List artifacts for a logged model.` - cmd.Long = `List artifacts for a logged model. - - List artifacts for a logged model. Takes an optional - artifact_directory_path prefix which if specified, the response contains - only artifacts with the specified prefix. - - Arguments: - MODEL_ID: The ID of the logged model for which to list the artifacts.` - - // This command is being previewed; hide from help output. - cmd.Hidden = true - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - listLoggedModelArtifactsReq.ModelId = args[0] - - response, err := w.Experiments.ListLoggedModelArtifacts(ctx, listLoggedModelArtifactsReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range listLoggedModelArtifactsOverrides { - fn(cmd, &listLoggedModelArtifactsReq) - } - - return cmd -} - // start log-batch command // Slice with functions to override default command behavior. @@ -1854,9 +1652,6 @@ func newLogLoggedModelParams() *cobra.Command { Arguments: MODEL_ID: The ID of the logged model to log params for.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -2110,9 +1905,6 @@ func newLogOutputs() *cobra.Command { Arguments: RUN_ID: The ID of the Run from which to log outputs.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -2625,9 +2417,6 @@ func newSearchLoggedModels() *cobra.Command { Search for Logged Models that satisfy specified search criteria.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -2863,9 +2652,6 @@ func newSetLoggedModelTags() *cobra.Command { Arguments: MODEL_ID: The ID of the logged model to set the tags on.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go index 1b58dbb81b..bc17c02bff 100755 --- a/cmd/workspace/genie/genie.go +++ b/cmd/workspace/genie/genie.go @@ -45,6 +45,7 @@ func New() *cobra.Command { cmd.AddCommand(newGetMessageQueryResult()) cmd.AddCommand(newGetMessageQueryResultByAttachment()) cmd.AddCommand(newGetSpace()) + cmd.AddCommand(newListSpaces()) cmd.AddCommand(newStartConversation()) // Apply optional overrides to this command. @@ -766,6 +767,65 @@ func newGetSpace() *cobra.Command { return cmd } +// start list-spaces command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listSpacesOverrides []func( + *cobra.Command, + *dashboards.GenieListSpacesRequest, +) + +func newListSpaces() *cobra.Command { + cmd := &cobra.Command{} + + var listSpacesReq dashboards.GenieListSpacesRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listSpacesReq.PageSize, "page-size", listSpacesReq.PageSize, `Maximum number of spaces to return per page.`) + cmd.Flags().StringVar(&listSpacesReq.PageToken, "page-token", listSpacesReq.PageToken, `Pagination token for getting the next page of results.`) + + cmd.Use = "list-spaces" + cmd.Short = `List Genie spaces.` + cmd.Long = `List Genie spaces. + + Get list of Genie Spaces.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response, err := w.Genie.ListSpaces(ctx, listSpacesReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listSpacesOverrides { + fn(cmd, &listSpacesReq) + } + + return cmd +} + // start start-conversation command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/grants/grants.go b/cmd/workspace/grants/grants.go index 9abbef1cf1..9fa89b72d5 100755 --- a/cmd/workspace/grants/grants.go +++ b/cmd/workspace/grants/grants.go @@ -3,8 +3,6 @@ package grants import ( - "fmt" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" @@ -68,36 +66,18 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&getReq.MaxResults, "max-results", getReq.MaxResults, `Specifies the maximum number of privileges to return (page length).`) + cmd.Flags().StringVar(&getReq.PageToken, "page-token", getReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Flags().StringVar(&getReq.Principal, "principal", getReq.Principal, `If provided, only the permissions for the specified principal (user or group) are returned.`) cmd.Use = "get SECURABLE_TYPE FULL_NAME" cmd.Short = `Get permissions.` cmd.Long = `Get permissions. - Gets the permissions for a securable. + Gets the permissions for a securable. Does not include inherited permissions. Arguments: - SECURABLE_TYPE: Type of securable. - Supported values: [ - CATALOG, - CLEAN_ROOM, - CONNECTION, - CREDENTIAL, - EXTERNAL_LOCATION, - EXTERNAL_METADATA, - FUNCTION, - METASTORE, - PIPELINE, - PROVIDER, - RECIPIENT, - SCHEMA, - SHARE, - STAGING_TABLE, - STORAGE_CREDENTIAL, - TABLE, - UNKNOWN_SECURABLE_TYPE, - VOLUME, - ] + SECURABLE_TYPE: Type of securable. FULL_NAME: Full name of securable.` cmd.Annotations = make(map[string]string) @@ -112,10 +92,7 @@ func newGet() *cobra.Command { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - _, err = fmt.Sscan(args[0], &getReq.SecurableType) - if err != nil { - return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) - } + getReq.SecurableType = args[0] getReq.FullName = args[1] response, err := w.Grants.Get(ctx, getReq) @@ -153,36 +130,19 @@ func newGetEffective() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&getEffectiveReq.MaxResults, "max-results", getEffectiveReq.MaxResults, `Specifies the maximum number of privileges to return (page length).`) + cmd.Flags().StringVar(&getEffectiveReq.PageToken, "page-token", getEffectiveReq.PageToken, `Opaque token for the next page of results (pagination).`) cmd.Flags().StringVar(&getEffectiveReq.Principal, "principal", getEffectiveReq.Principal, `If provided, only the effective permissions for the specified principal (user or group) are returned.`) cmd.Use = "get-effective SECURABLE_TYPE FULL_NAME" cmd.Short = `Get effective permissions.` cmd.Long = `Get effective permissions. - Gets the effective permissions for a securable. + Gets the effective permissions for a securable. Includes inherited permissions + from any parent securables. Arguments: - SECURABLE_TYPE: Type of securable. - Supported values: [ - CATALOG, - CLEAN_ROOM, - CONNECTION, - CREDENTIAL, - EXTERNAL_LOCATION, - EXTERNAL_METADATA, - FUNCTION, - METASTORE, - PIPELINE, - PROVIDER, - RECIPIENT, - SCHEMA, - SHARE, - STAGING_TABLE, - STORAGE_CREDENTIAL, - TABLE, - UNKNOWN_SECURABLE_TYPE, - VOLUME, - ] + SECURABLE_TYPE: Type of securable. FULL_NAME: Full name of securable.` cmd.Annotations = make(map[string]string) @@ -197,10 +157,7 @@ func newGetEffective() *cobra.Command { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - _, err = fmt.Sscan(args[0], &getEffectiveReq.SecurableType) - if err != nil { - return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) - } + getEffectiveReq.SecurableType = args[0] getEffectiveReq.FullName = args[1] response, err := w.Grants.GetEffective(ctx, getEffectiveReq) @@ -249,27 +206,7 @@ func newUpdate() *cobra.Command { Updates the permissions for a securable. Arguments: - SECURABLE_TYPE: Type of securable. - Supported values: [ - CATALOG, - CLEAN_ROOM, - CONNECTION, - CREDENTIAL, - EXTERNAL_LOCATION, - EXTERNAL_METADATA, - FUNCTION, - METASTORE, - PIPELINE, - PROVIDER, - RECIPIENT, - SCHEMA, - SHARE, - STAGING_TABLE, - STORAGE_CREDENTIAL, - TABLE, - UNKNOWN_SECURABLE_TYPE, - VOLUME, - ] + SECURABLE_TYPE: Type of securable. FULL_NAME: Full name of securable.` cmd.Annotations = make(map[string]string) @@ -296,10 +233,7 @@ func newUpdate() *cobra.Command { } } } - _, err = fmt.Sscan(args[0], &updateReq.SecurableType) - if err != nil { - return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) - } + updateReq.SecurableType = args[0] updateReq.FullName = args[1] response, err := w.Grants.Update(ctx, updateReq) diff --git a/cmd/workspace/groups.go b/cmd/workspace/groups.go index 8827682fa6..817f915345 100644 --- a/cmd/workspace/groups.go +++ b/cmd/workspace/groups.go @@ -76,5 +76,17 @@ func Groups() []cobra.Group { ID: "cleanrooms", Title: "Clean Rooms", }, + { + ID: "aibuilder", + Title: "AI Builder", + }, + { + ID: "database", + Title: "Database", + }, + { + ID: "qualitymonitorv2", + Title: "Quality Monitor v2", + }, } } diff --git a/cmd/workspace/lakeview-embedded/lakeview-embedded.go b/cmd/workspace/lakeview-embedded/lakeview-embedded.go index 782b4effcf..06ed7f1f73 100755 --- a/cmd/workspace/lakeview-embedded/lakeview-embedded.go +++ b/cmd/workspace/lakeview-embedded/lakeview-embedded.go @@ -27,7 +27,6 @@ func New() *cobra.Command { } // Add methods - cmd.AddCommand(newGetPublishedDashboardEmbedded()) cmd.AddCommand(newGetPublishedDashboardTokenInfo()) // Apply optional overrides to this command. @@ -38,67 +37,6 @@ func New() *cobra.Command { return cmd } -// start get-published-dashboard-embedded command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var getPublishedDashboardEmbeddedOverrides []func( - *cobra.Command, - *dashboards.GetPublishedDashboardEmbeddedRequest, -) - -func newGetPublishedDashboardEmbedded() *cobra.Command { - cmd := &cobra.Command{} - - var getPublishedDashboardEmbeddedReq dashboards.GetPublishedDashboardEmbeddedRequest - - // TODO: short flags - - cmd.Use = "get-published-dashboard-embedded DASHBOARD_ID" - cmd.Short = `Read a published dashboard in an embedded ui.` - cmd.Long = `Read a published dashboard in an embedded ui. - - Get the current published dashboard within an embedded context. - - Arguments: - DASHBOARD_ID: UUID identifying the published dashboard.` - - // This command is being previewed; hide from help output. - cmd.Hidden = true - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - getPublishedDashboardEmbeddedReq.DashboardId = args[0] - - err = w.LakeviewEmbedded.GetPublishedDashboardEmbedded(ctx, getPublishedDashboardEmbeddedReq) - if err != nil { - return err - } - return nil - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range getPublishedDashboardEmbeddedOverrides { - fn(cmd, &getPublishedDashboardEmbeddedReq) - } - - return cmd -} - // start get-published-dashboard-token-info command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/metastores/metastores.go b/cmd/workspace/metastores/metastores.go index 99dc7cfbb3..fe5bfb5178 100755 --- a/cmd/workspace/metastores/metastores.go +++ b/cmd/workspace/metastores/metastores.go @@ -90,9 +90,9 @@ func newAssign() *cobra.Command { Arguments: WORKSPACE_ID: A workspace ID. METASTORE_ID: The unique ID of the metastore. - DEFAULT_CATALOG_NAME: The name of the default catalog in the metastore. This field is depracted. - Please use "Default Namespace API" to configure the default catalog for a - Databricks workspace.` + DEFAULT_CATALOG_NAME: The name of the default catalog in the metastore. This field is + deprecated. Please use "Default Namespace API" to configure the default + catalog for a Databricks workspace.` cmd.Annotations = make(map[string]string) @@ -314,28 +314,16 @@ func newDelete() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ID argument specified. Loading names for Metastores drop-down." - names, err := w.Metastores.MetastoreInfoNameToMetastoreIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Metastores drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Unique ID of the metastore") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have unique id of the metastore") - } deleteReq.Id = args[0] err = w.Metastores.Delete(ctx, deleteReq) @@ -385,28 +373,16 @@ func newGet() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ID argument specified. Loading names for Metastores drop-down." - names, err := w.Metastores.MetastoreInfoNameToMetastoreIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Metastores drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Unique ID of the metastore") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have unique id of the metastore") - } getReq.Id = args[0] response, err := w.Metastores.Get(ctx, getReq) @@ -434,11 +410,19 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *catalog.ListMetastoresRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq catalog.ListMetastoresRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of metastores to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list" cmd.Short = `List metastores.` cmd.Long = `List metastores. @@ -449,11 +433,17 @@ func newList() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - response := w.Metastores.List(ctx) + + response := w.Metastores.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -463,7 +453,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -614,6 +604,11 @@ func newUpdate() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -631,23 +626,6 @@ func newUpdate() *cobra.Command { } } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ID argument specified. Loading names for Metastores drop-down." - names, err := w.Metastores.MetastoreInfoNameToMetastoreIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Metastores drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Unique ID of the metastore") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have unique id of the metastore") - } updateReq.Id = args[0] response, err := w.Metastores.Update(ctx, updateReq) @@ -704,6 +682,11 @@ func newUpdateAssignment() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -721,23 +704,6 @@ func newUpdateAssignment() *cobra.Command { } } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No WORKSPACE_ID argument specified. Loading names for Metastores drop-down." - names, err := w.Metastores.MetastoreInfoNameToMetastoreIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Metastores drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "A workspace ID") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have a workspace id") - } _, err = fmt.Sscan(args[0], &updateAssignmentReq.WorkspaceId) if err != nil { return fmt.Errorf("invalid WORKSPACE_ID: %s", args[0]) diff --git a/cmd/workspace/metastores/overrides.go b/cmd/workspace/metastores/overrides.go index 3ee6a10714..4f81c5ce5b 100644 --- a/cmd/workspace/metastores/overrides.go +++ b/cmd/workspace/metastores/overrides.go @@ -2,10 +2,11 @@ package metastores import ( "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/spf13/cobra" ) -func listOverride(listCmd *cobra.Command) { +func listOverride(listCmd *cobra.Command, req *catalog.ListMetastoresRequest) { listCmd.Annotations["headerTemplate"] = cmdio.Heredoc(` {{header "ID"}} {{header "Name"}} {{"Region"}}`) listCmd.Annotations["template"] = cmdio.Heredoc(` diff --git a/cmd/workspace/model-registry/model-registry.go b/cmd/workspace/model-registry/model-registry.go index 4fbabec8f0..a5a41d7f59 100755 --- a/cmd/workspace/model-registry/model-registry.go +++ b/cmd/workspace/model-registry/model-registry.go @@ -585,7 +585,7 @@ func newCreateWebhook() *cobra.Command { cmd.Flags().StringVar(&createWebhookReq.Description, "description", createWebhookReq.Description, `User-specified description for the webhook.`) // TODO: complex arg: http_url_spec // TODO: complex arg: job_spec - cmd.Flags().StringVar(&createWebhookReq.ModelName, "model-name", createWebhookReq.ModelName, `Name of the model whose events would trigger this webhook.`) + cmd.Flags().StringVar(&createWebhookReq.ModelName, "model-name", createWebhookReq.ModelName, `If model name is not specified, a registry-wide webhook is created that listens for the specified events across all versions of all registered models.`) cmd.Flags().Var(&createWebhookReq.Status, "status", `Enable or disable triggering the webhook, or put the webhook into test mode. Supported values: [ACTIVE, DISABLED, TEST_MODE]`) cmd.Use = "create-webhook" @@ -657,7 +657,10 @@ func newDeleteComment() *cobra.Command { cmd.Short = `Delete a comment.` cmd.Long = `Delete a comment. - Deletes a comment on a model version.` + Deletes a comment on a model version. + + Arguments: + ID: Unique identifier of an activity` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index 3000842482..29a9a7d5fe 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -153,7 +153,8 @@ func newDelete() *cobra.Command { cmd.Short = `Delete a pipeline.` cmd.Long = `Delete a pipeline. - Deletes a pipeline.` + Deletes a pipeline. Deleting a pipeline is a permanent action that stops and + removes the pipeline and its tables. You cannot undo this action.` cmd.Annotations = make(map[string]string) @@ -980,6 +981,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.Schema, "schema", updateReq.Schema, `The default schema (database) where tables are read from or published to.`) cmd.Flags().BoolVar(&updateReq.Serverless, "serverless", updateReq.Serverless, `Whether serverless compute is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.Storage, "storage", updateReq.Storage, `DBFS root directory for storing checkpoints and tables.`) + // TODO: map via StringToStringVar: tags cmd.Flags().StringVar(&updateReq.Target, "target", updateReq.Target, `Target schema (database) to add tables in this pipeline to.`) // TODO: complex arg: trigger diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index b91638478d..5228982aec 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -79,7 +79,7 @@ func newCreate() *cobra.Command { Arguments: NAME: The name of the Provider. AUTHENTICATION_TYPE: The delta sharing authentication type. - Supported values: [DATABRICKS, OAUTH_CLIENT_CREDENTIALS, TOKEN]` + Supported values: [DATABRICKS, OAUTH_CLIENT_CREDENTIALS, OIDC_FEDERATION, TOKEN]` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/quality-monitor-v2/quality-monitor-v2.go b/cmd/workspace/quality-monitor-v2/quality-monitor-v2.go new file mode 100755 index 0000000000..ea0175fdd5 --- /dev/null +++ b/cmd/workspace/quality-monitor-v2/quality-monitor-v2.go @@ -0,0 +1,400 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package quality_monitor_v2 + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/qualitymonitorv2" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "quality-monitor-v2", + Short: `Manage data quality of UC objects (currently support schema).`, + Long: `Manage data quality of UC objects (currently support schema)`, + GroupID: "qualitymonitorv2", + Annotations: map[string]string{ + "package": "qualitymonitorv2", + }, + RunE: root.ReportUnknownSubcommand, + } + + // Add methods + cmd.AddCommand(newCreateQualityMonitor()) + cmd.AddCommand(newDeleteQualityMonitor()) + cmd.AddCommand(newGetQualityMonitor()) + cmd.AddCommand(newListQualityMonitor()) + cmd.AddCommand(newUpdateQualityMonitor()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create-quality-monitor command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createQualityMonitorOverrides []func( + *cobra.Command, + *qualitymonitorv2.CreateQualityMonitorRequest, +) + +func newCreateQualityMonitor() *cobra.Command { + cmd := &cobra.Command{} + + var createQualityMonitorReq qualitymonitorv2.CreateQualityMonitorRequest + createQualityMonitorReq.QualityMonitor = qualitymonitorv2.QualityMonitor{} + var createQualityMonitorJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createQualityMonitorJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: anomaly_detection_config + + cmd.Use = "create-quality-monitor OBJECT_TYPE OBJECT_ID" + cmd.Short = `Create a quality monitor.` + cmd.Long = `Create a quality monitor. + + Create a quality monitor on UC object + + Arguments: + OBJECT_TYPE: The type of the monitored object. Can be one of the following: schema. + OBJECT_ID: The uuid of the request object. For example, schema id.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'object_type', 'object_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createQualityMonitorJson.Unmarshal(&createQualityMonitorReq.QualityMonitor) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + if !cmd.Flags().Changed("json") { + createQualityMonitorReq.QualityMonitor.ObjectType = args[0] + } + if !cmd.Flags().Changed("json") { + createQualityMonitorReq.QualityMonitor.ObjectId = args[1] + } + + response, err := w.QualityMonitorV2.CreateQualityMonitor(ctx, createQualityMonitorReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createQualityMonitorOverrides { + fn(cmd, &createQualityMonitorReq) + } + + return cmd +} + +// start delete-quality-monitor command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteQualityMonitorOverrides []func( + *cobra.Command, + *qualitymonitorv2.DeleteQualityMonitorRequest, +) + +func newDeleteQualityMonitor() *cobra.Command { + cmd := &cobra.Command{} + + var deleteQualityMonitorReq qualitymonitorv2.DeleteQualityMonitorRequest + + // TODO: short flags + + cmd.Use = "delete-quality-monitor OBJECT_TYPE OBJECT_ID" + cmd.Short = `Delete a quality monitor.` + cmd.Long = `Delete a quality monitor. + + Delete a quality monitor on UC object + + Arguments: + OBJECT_TYPE: The type of the monitored object. Can be one of the following: schema. + OBJECT_ID: The uuid of the request object. For example, schema id.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + deleteQualityMonitorReq.ObjectType = args[0] + deleteQualityMonitorReq.ObjectId = args[1] + + err = w.QualityMonitorV2.DeleteQualityMonitor(ctx, deleteQualityMonitorReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteQualityMonitorOverrides { + fn(cmd, &deleteQualityMonitorReq) + } + + return cmd +} + +// start get-quality-monitor command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getQualityMonitorOverrides []func( + *cobra.Command, + *qualitymonitorv2.GetQualityMonitorRequest, +) + +func newGetQualityMonitor() *cobra.Command { + cmd := &cobra.Command{} + + var getQualityMonitorReq qualitymonitorv2.GetQualityMonitorRequest + + // TODO: short flags + + cmd.Use = "get-quality-monitor OBJECT_TYPE OBJECT_ID" + cmd.Short = `Read a quality monitor.` + cmd.Long = `Read a quality monitor. + + Read a quality monitor on UC object + + Arguments: + OBJECT_TYPE: The type of the monitored object. Can be one of the following: schema. + OBJECT_ID: The uuid of the request object. For example, schema id.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getQualityMonitorReq.ObjectType = args[0] + getQualityMonitorReq.ObjectId = args[1] + + response, err := w.QualityMonitorV2.GetQualityMonitor(ctx, getQualityMonitorReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getQualityMonitorOverrides { + fn(cmd, &getQualityMonitorReq) + } + + return cmd +} + +// start list-quality-monitor command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listQualityMonitorOverrides []func( + *cobra.Command, + *qualitymonitorv2.ListQualityMonitorRequest, +) + +func newListQualityMonitor() *cobra.Command { + cmd := &cobra.Command{} + + var listQualityMonitorReq qualitymonitorv2.ListQualityMonitorRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listQualityMonitorReq.PageSize, "page-size", listQualityMonitorReq.PageSize, ``) + cmd.Flags().StringVar(&listQualityMonitorReq.PageToken, "page-token", listQualityMonitorReq.PageToken, ``) + + cmd.Use = "list-quality-monitor" + cmd.Short = `List quality monitors.` + cmd.Long = `List quality monitors. + + (Unimplemented) List quality monitors` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response := w.QualityMonitorV2.ListQualityMonitor(ctx, listQualityMonitorReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listQualityMonitorOverrides { + fn(cmd, &listQualityMonitorReq) + } + + return cmd +} + +// start update-quality-monitor command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateQualityMonitorOverrides []func( + *cobra.Command, + *qualitymonitorv2.UpdateQualityMonitorRequest, +) + +func newUpdateQualityMonitor() *cobra.Command { + cmd := &cobra.Command{} + + var updateQualityMonitorReq qualitymonitorv2.UpdateQualityMonitorRequest + updateQualityMonitorReq.QualityMonitor = qualitymonitorv2.QualityMonitor{} + var updateQualityMonitorJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateQualityMonitorJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: anomaly_detection_config + + cmd.Use = "update-quality-monitor OBJECT_TYPE OBJECT_ID OBJECT_TYPE OBJECT_ID" + cmd.Short = `Update a quality monitor.` + cmd.Long = `Update a quality monitor. + + (Unimplemented) Update a quality monitor on UC object + + Arguments: + OBJECT_TYPE: The type of the monitored object. Can be one of the following: schema. + OBJECT_ID: The uuid of the request object. For example, schema id. + OBJECT_TYPE: The type of the monitored object. Can be one of the following: schema. + OBJECT_ID: The uuid of the request object. For example, schema id.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only OBJECT_TYPE, OBJECT_ID as positional arguments. Provide 'object_type', 'object_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(4) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateQualityMonitorJson.Unmarshal(&updateQualityMonitorReq.QualityMonitor) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + updateQualityMonitorReq.ObjectType = args[0] + updateQualityMonitorReq.ObjectId = args[1] + if !cmd.Flags().Changed("json") { + updateQualityMonitorReq.QualityMonitor.ObjectType = args[2] + } + if !cmd.Flags().Changed("json") { + updateQualityMonitorReq.QualityMonitor.ObjectId = args[3] + } + + response, err := w.QualityMonitorV2.UpdateQualityMonitor(ctx, updateQualityMonitorReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateQualityMonitorOverrides { + fn(cmd, &updateQualityMonitorReq) + } + + return cmd +} + +// end service QualityMonitorV2 diff --git a/cmd/workspace/query-execution/query-execution.go b/cmd/workspace/query-execution/query-execution.go old mode 100755 new mode 100644 index 63d57bba3d..119de7857e --- a/cmd/workspace/query-execution/query-execution.go +++ b/cmd/workspace/query-execution/query-execution.go @@ -1,247 +1,2 @@ -// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. - +// Package query_execution provides functionality for query execution in Databricks. package query_execution - -import ( - "fmt" - - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/flags" - "github.com/databricks/databricks-sdk-go/service/dashboards" - "github.com/spf13/cobra" -) - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var cmdOverrides []func(*cobra.Command) - -func New() *cobra.Command { - cmd := &cobra.Command{ - Use: "query-execution", - Short: `Query execution APIs for AI / BI Dashboards.`, - Long: `Query execution APIs for AI / BI Dashboards`, - GroupID: "dashboards", - Annotations: map[string]string{ - "package": "dashboards", - }, - - // This service is being previewed; hide from help output. - Hidden: true, - RunE: root.ReportUnknownSubcommand, - } - - // Add methods - cmd.AddCommand(newCancelPublishedQueryExecution()) - cmd.AddCommand(newExecutePublishedDashboardQuery()) - cmd.AddCommand(newPollPublishedQueryStatus()) - - // Apply optional overrides to this command. - for _, fn := range cmdOverrides { - fn(cmd) - } - - return cmd -} - -// start cancel-published-query-execution command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var cancelPublishedQueryExecutionOverrides []func( - *cobra.Command, - *dashboards.CancelPublishedQueryExecutionRequest, -) - -func newCancelPublishedQueryExecution() *cobra.Command { - cmd := &cobra.Command{} - - var cancelPublishedQueryExecutionReq dashboards.CancelPublishedQueryExecutionRequest - - // TODO: short flags - - // TODO: array: tokens - - cmd.Use = "cancel-published-query-execution DASHBOARD_NAME DASHBOARD_REVISION_ID" - cmd.Short = `Cancel the results for the a query for a published, embedded dashboard.` - cmd.Long = `Cancel the results for the a query for a published, embedded dashboard.` - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(2) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - cancelPublishedQueryExecutionReq.DashboardName = args[0] - cancelPublishedQueryExecutionReq.DashboardRevisionId = args[1] - - response, err := w.QueryExecution.CancelPublishedQueryExecution(ctx, cancelPublishedQueryExecutionReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range cancelPublishedQueryExecutionOverrides { - fn(cmd, &cancelPublishedQueryExecutionReq) - } - - return cmd -} - -// start execute-published-dashboard-query command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var executePublishedDashboardQueryOverrides []func( - *cobra.Command, - *dashboards.ExecutePublishedDashboardQueryRequest, -) - -func newExecutePublishedDashboardQuery() *cobra.Command { - cmd := &cobra.Command{} - - var executePublishedDashboardQueryReq dashboards.ExecutePublishedDashboardQueryRequest - var executePublishedDashboardQueryJson flags.JsonFlag - - // TODO: short flags - cmd.Flags().Var(&executePublishedDashboardQueryJson, "json", `either inline JSON string or @path/to/file.json with request body`) - - cmd.Flags().StringVar(&executePublishedDashboardQueryReq.OverrideWarehouseId, "override-warehouse-id", executePublishedDashboardQueryReq.OverrideWarehouseId, `A dashboard schedule can override the warehouse used as compute for processing the published dashboard queries.`) - - cmd.Use = "execute-published-dashboard-query DASHBOARD_NAME DASHBOARD_REVISION_ID" - cmd.Short = `Execute a query for a published dashboard.` - cmd.Long = `Execute a query for a published dashboard. - - Arguments: - DASHBOARD_NAME: Dashboard name and revision_id is required to retrieve - PublishedDatasetDataModel which contains the list of datasets, - warehouse_id, and embedded_credentials - DASHBOARD_REVISION_ID: ` - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("json") { - err := root.ExactArgs(0)(cmd, args) - if err != nil { - return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'dashboard_name', 'dashboard_revision_id' in your JSON input") - } - return nil - } - check := root.ExactArgs(2) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - if cmd.Flags().Changed("json") { - diags := executePublishedDashboardQueryJson.Unmarshal(&executePublishedDashboardQueryReq) - if diags.HasError() { - return diags.Error() - } - if len(diags) > 0 { - err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) - if err != nil { - return err - } - } - } - if !cmd.Flags().Changed("json") { - executePublishedDashboardQueryReq.DashboardName = args[0] - } - if !cmd.Flags().Changed("json") { - executePublishedDashboardQueryReq.DashboardRevisionId = args[1] - } - - err = w.QueryExecution.ExecutePublishedDashboardQuery(ctx, executePublishedDashboardQueryReq) - if err != nil { - return err - } - return nil - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range executePublishedDashboardQueryOverrides { - fn(cmd, &executePublishedDashboardQueryReq) - } - - return cmd -} - -// start poll-published-query-status command - -// Slice with functions to override default command behavior. -// Functions can be added from the `init()` function in manually curated files in this directory. -var pollPublishedQueryStatusOverrides []func( - *cobra.Command, - *dashboards.PollPublishedQueryStatusRequest, -) - -func newPollPublishedQueryStatus() *cobra.Command { - cmd := &cobra.Command{} - - var pollPublishedQueryStatusReq dashboards.PollPublishedQueryStatusRequest - - // TODO: short flags - - // TODO: array: tokens - - cmd.Use = "poll-published-query-status DASHBOARD_NAME DASHBOARD_REVISION_ID" - cmd.Short = `Poll the results for the a query for a published, embedded dashboard.` - cmd.Long = `Poll the results for the a query for a published, embedded dashboard.` - - cmd.Annotations = make(map[string]string) - - cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(2) - return check(cmd, args) - } - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := cmdctx.WorkspaceClient(ctx) - - pollPublishedQueryStatusReq.DashboardName = args[0] - pollPublishedQueryStatusReq.DashboardRevisionId = args[1] - - response, err := w.QueryExecution.PollPublishedQueryStatus(ctx, pollPublishedQueryStatusReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) - } - - // Disable completions since they are not applicable. - // Can be overridden by manual implementation in `override.go`. - cmd.ValidArgsFunction = cobra.NoFileCompletions - - // Apply optional overrides to this command. - for _, fn := range pollPublishedQueryStatusOverrides { - fn(cmd, &pollPublishedQueryStatusReq) - } - - return cmd -} - -// end service QueryExecution diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go index 8b036de7af..1864707766 100755 --- a/cmd/workspace/recipients/recipients.go +++ b/cmd/workspace/recipients/recipients.go @@ -99,7 +99,7 @@ func newCreate() *cobra.Command { Arguments: NAME: Name of Recipient. AUTHENTICATION_TYPE: The delta sharing authentication type. - Supported values: [DATABRICKS, OAUTH_CLIENT_CREDENTIALS, TOKEN]` + Supported values: [DATABRICKS, OAUTH_CLIENT_CREDENTIALS, OIDC_FEDERATION, TOKEN]` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/schemas/schemas.go b/cmd/workspace/schemas/schemas.go index bcd3273c66..9c3425d6b1 100755 --- a/cmd/workspace/schemas/schemas.go +++ b/cmd/workspace/schemas/schemas.go @@ -169,28 +169,16 @@ func newDelete() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No FULL_NAME argument specified. Loading names for Schemas drop-down." - names, err := w.Schemas.SchemaInfoNameToFullNameMap(ctx, catalog.ListSchemasRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Schemas drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Full name of the schema") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have full name of the schema") - } deleteReq.FullName = args[0] err = w.Schemas.Delete(ctx, deleteReq) @@ -243,28 +231,16 @@ func newGet() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No FULL_NAME argument specified. Loading names for Schemas drop-down." - names, err := w.Schemas.SchemaInfoNameToFullNameMap(ctx, catalog.ListSchemasRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Schemas drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Full name of the schema") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have full name of the schema") - } getReq.FullName = args[0] response, err := w.Schemas.Get(ctx, getReq) @@ -368,7 +344,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `User-provided free-form text description.`) - cmd.Flags().Var(&updateReq.EnablePredictiveOptimization, "enable-predictive-optimization", `. Supported values: [DISABLE, ENABLE, INHERIT]`) + cmd.Flags().Var(&updateReq.EnablePredictiveOptimization, "enable-predictive-optimization", `Whether predictive optimization should be enabled for this object and objects under it. Supported values: [DISABLE, ENABLE, INHERIT]`) cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the schema.`) cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of schema.`) // TODO: map via StringToStringVar: properties @@ -388,6 +364,11 @@ func newUpdate() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -405,23 +386,6 @@ func newUpdate() *cobra.Command { } } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No FULL_NAME argument specified. Loading names for Schemas drop-down." - names, err := w.Schemas.SchemaInfoNameToFullNameMap(ctx, catalog.ListSchemasRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Schemas drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Full name of the schema") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have full name of the schema") - } updateReq.FullName = args[0] response, err := w.Schemas.Update(ctx, updateReq) diff --git a/cmd/workspace/settings/settings.go b/cmd/workspace/settings/settings.go index 50519f2adf..2754412a77 100755 --- a/cmd/workspace/settings/settings.go +++ b/cmd/workspace/settings/settings.go @@ -10,6 +10,7 @@ import ( aibi_dashboard_embedding_approved_domains "github.com/databricks/cli/cmd/workspace/aibi-dashboard-embedding-approved-domains" automatic_cluster_update "github.com/databricks/cli/cmd/workspace/automatic-cluster-update" compliance_security_profile "github.com/databricks/cli/cmd/workspace/compliance-security-profile" + dashboard_email_subscriptions "github.com/databricks/cli/cmd/workspace/dashboard-email-subscriptions" default_namespace "github.com/databricks/cli/cmd/workspace/default-namespace" disable_legacy_access "github.com/databricks/cli/cmd/workspace/disable-legacy-access" disable_legacy_dbfs "github.com/databricks/cli/cmd/workspace/disable-legacy-dbfs" @@ -19,6 +20,7 @@ import ( enhanced_security_monitoring "github.com/databricks/cli/cmd/workspace/enhanced-security-monitoring" llm_proxy_partner_powered_workspace "github.com/databricks/cli/cmd/workspace/llm-proxy-partner-powered-workspace" restrict_workspace_admins "github.com/databricks/cli/cmd/workspace/restrict-workspace-admins" + sql_results_download "github.com/databricks/cli/cmd/workspace/sql-results-download" ) // Slice with functions to override default command behavior. @@ -42,6 +44,7 @@ func New() *cobra.Command { cmd.AddCommand(aibi_dashboard_embedding_approved_domains.New()) cmd.AddCommand(automatic_cluster_update.New()) cmd.AddCommand(compliance_security_profile.New()) + cmd.AddCommand(dashboard_email_subscriptions.New()) cmd.AddCommand(default_namespace.New()) cmd.AddCommand(disable_legacy_access.New()) cmd.AddCommand(disable_legacy_dbfs.New()) @@ -51,6 +54,7 @@ func New() *cobra.Command { cmd.AddCommand(enhanced_security_monitoring.New()) cmd.AddCommand(llm_proxy_partner_powered_workspace.New()) cmd.AddCommand(restrict_workspace_admins.New()) + cmd.AddCommand(sql_results_download.New()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { diff --git a/cmd/workspace/sql-results-download/sql-results-download.go b/cmd/workspace/sql-results-download/sql-results-download.go new file mode 100755 index 0000000000..b807a767a2 --- /dev/null +++ b/cmd/workspace/sql-results-download/sql-results-download.go @@ -0,0 +1,218 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package sql_results_download + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "sql-results-download", + Short: `Controls whether users within the workspace are allowed to download results from the SQL Editor and AI/BI Dashboards UIs.`, + Long: `Controls whether users within the workspace are allowed to download results + from the SQL Editor and AI/BI Dashboards UIs. By default, this setting is + enabled (set to true)`, + RunE: root.ReportUnknownSubcommand, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteSqlResultsDownloadRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteSqlResultsDownloadRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete the SQL Results Download setting.` + cmd.Long = `Delete the SQL Results Download setting. + + Reverts the SQL Results Download setting to its default value.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response, err := w.Settings.SqlResultsDownload().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetSqlResultsDownloadRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetSqlResultsDownloadRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Get the SQL Results Download setting.` + cmd.Long = `Get the SQL Results Download setting. + + Gets the SQL Results Download setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response, err := w.Settings.SqlResultsDownload().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateSqlResultsDownloadRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateSqlResultsDownloadRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update the SQL Results Download setting.` + cmd.Long = `Update the SQL Results Download setting. + + Updates the SQL Results Download setting.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.Settings.SqlResultsDownload().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service SqlResultsDownload diff --git a/experimental/python/README.md b/experimental/python/README.md index cec677e15f..28fbc2fda5 100644 --- a/experimental/python/README.md +++ b/experimental/python/README.md @@ -13,7 +13,7 @@ Reference documentation is available at https://databricks.github.io/cli/experim To use `databricks-bundles`, you must first: -1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.254.0 or above +1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.255.0 or above 2. Authenticate to your Databricks workspace if you have not done so already: ```bash diff --git a/experimental/python/databricks/bundles/compute/_models/environment.py b/experimental/python/databricks/bundles/compute/_models/environment.py index 16cc9d7140..c8bdee0917 100644 --- a/experimental/python/databricks/bundles/compute/_models/environment.py +++ b/experimental/python/databricks/bundles/compute/_models/environment.py @@ -3,11 +3,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import ( - VariableOr, - VariableOrList, - VariableOrOptional, -) +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional if TYPE_CHECKING: from typing_extensions import Self @@ -20,14 +16,6 @@ class Environment: In this minimal environment spec, only pip dependencies are supported. """ - client: VariableOr[str] - """ - Client version used by the environment - The client is the user-facing environment of the runtime. - Each client comes with a specific set of pre-installed libraries. - The version is a string, consisting of the major client version. - """ - dependencies: VariableOrList[str] = field(default_factory=list) """ List of pip dependencies, as supported by the version of pip in this environment. @@ -35,12 +23,9 @@ class Environment: environment_version: VariableOrOptional[str] = None """ - :meta private: [EXPERIMENTAL] - - We renamed `client` to `environment_version` in notebook exports. This field is meant solely so that imported notebooks with `environment_version` can be deserialized - correctly, in a backwards-compatible way (i.e. if `client` is specified instead of `environment_version`, it will be deserialized correctly). Do NOT use this field - for any other purpose, e.g. notebook storage. - This field is not yet exposed to customers (e.g. in the jobs API). + Required. Environment version used by the environment. + Each version comes with a specific Python version and a set of Python packages. + The version is a string, consisting of an integer. """ jar_dependencies: VariableOrList[str] = field(default_factory=list) @@ -61,14 +46,6 @@ def as_dict(self) -> "EnvironmentDict": class EnvironmentDict(TypedDict, total=False): """""" - client: VariableOr[str] - """ - Client version used by the environment - The client is the user-facing environment of the runtime. - Each client comes with a specific set of pre-installed libraries. - The version is a string, consisting of the major client version. - """ - dependencies: VariableOrList[str] """ List of pip dependencies, as supported by the version of pip in this environment. @@ -76,12 +53,9 @@ class EnvironmentDict(TypedDict, total=False): environment_version: VariableOrOptional[str] """ - :meta private: [EXPERIMENTAL] - - We renamed `client` to `environment_version` in notebook exports. This field is meant solely so that imported notebooks with `environment_version` can be deserialized - correctly, in a backwards-compatible way (i.e. if `client` is specified instead of `environment_version`, it will be deserialized correctly). Do NOT use this field - for any other purpose, e.g. notebook storage. - This field is not yet exposed to customers (e.g. in the jobs API). + Required. Environment version used by the environment. + Each version comes with a specific Python version and a set of Python packages. + The version is a string, consisting of an integer. """ jar_dependencies: VariableOrList[str] diff --git a/experimental/python/databricks/bundles/jobs/__init__.py b/experimental/python/databricks/bundles/jobs/__init__.py index 32fd0e6599..3eb3d43185 100644 --- a/experimental/python/databricks/bundles/jobs/__init__.py +++ b/experimental/python/databricks/bundles/jobs/__init__.py @@ -53,6 +53,9 @@ "DbfsStorageInfo", "DbfsStorageInfoDict", "DbfsStorageInfoParam", + "DbtCloudTask", + "DbtCloudTaskDict", + "DbtCloudTaskParam", "DbtTask", "DbtTaskDict", "DbtTaskParam", @@ -445,6 +448,11 @@ DashboardTaskDict, DashboardTaskParam, ) +from databricks.bundles.jobs._models.dbt_cloud_task import ( + DbtCloudTask, + DbtCloudTaskDict, + DbtCloudTaskParam, +) from databricks.bundles.jobs._models.dbt_task import DbtTask, DbtTaskDict, DbtTaskParam from databricks.bundles.jobs._models.file_arrival_trigger_configuration import ( FileArrivalTriggerConfiguration, diff --git a/experimental/python/databricks/bundles/jobs/_models/dashboard_task.py b/experimental/python/databricks/bundles/jobs/_models/dashboard_task.py index 6284ca36d3..b42ef0bdd5 100644 --- a/experimental/python/databricks/bundles/jobs/_models/dashboard_task.py +++ b/experimental/python/databricks/bundles/jobs/_models/dashboard_task.py @@ -4,10 +4,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrOptional -from databricks.bundles.jobs._models.subscription import ( - Subscription, - SubscriptionParam, -) +from databricks.bundles.jobs._models.subscription import Subscription, SubscriptionParam if TYPE_CHECKING: from typing_extensions import Self diff --git a/experimental/python/databricks/bundles/jobs/_models/dbt_cloud_task.py b/experimental/python/databricks/bundles/jobs/_models/dbt_cloud_task.py new file mode 100644 index 0000000000..d1d862c7ef --- /dev/null +++ b/experimental/python/databricks/bundles/jobs/_models/dbt_cloud_task.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DbtCloudTask: + """ + :meta private: [EXPERIMENTAL] + """ + + connection_resource_name: VariableOrOptional[str] = None + """ + The resource name of the UC connection that authenticates the dbt Cloud for this task + """ + + dbt_cloud_job_id: VariableOrOptional[int] = None + """ + Id of the dbt Cloud job to be triggered + """ + + @classmethod + def from_dict(cls, value: "DbtCloudTaskDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DbtCloudTaskDict": + return _transform_to_json_value(self) # type:ignore + + +class DbtCloudTaskDict(TypedDict, total=False): + """""" + + connection_resource_name: VariableOrOptional[str] + """ + The resource name of the UC connection that authenticates the dbt Cloud for this task + """ + + dbt_cloud_job_id: VariableOrOptional[int] + """ + Id of the dbt Cloud job to be triggered + """ + + +DbtCloudTaskParam = DbtCloudTaskDict | DbtCloudTask diff --git a/experimental/python/databricks/bundles/jobs/_models/job.py b/experimental/python/databricks/bundles/jobs/_models/job.py index c72a20a329..ca40311133 100644 --- a/experimental/python/databricks/bundles/jobs/_models/job.py +++ b/experimental/python/databricks/bundles/jobs/_models/job.py @@ -17,10 +17,7 @@ CronSchedule, CronScheduleParam, ) -from databricks.bundles.jobs._models.git_source import ( - GitSource, - GitSourceParam, -) +from databricks.bundles.jobs._models.git_source import GitSource, GitSourceParam from databricks.bundles.jobs._models.job_cluster import JobCluster, JobClusterParam from databricks.bundles.jobs._models.job_email_notifications import ( JobEmailNotifications, diff --git a/experimental/python/databricks/bundles/jobs/_models/task.py b/experimental/python/databricks/bundles/jobs/_models/task.py index 7120c970a8..8da07a4ab3 100644 --- a/experimental/python/databricks/bundles/jobs/_models/task.py +++ b/experimental/python/databricks/bundles/jobs/_models/task.py @@ -28,6 +28,10 @@ DashboardTask, DashboardTaskParam, ) +from databricks.bundles.jobs._models.dbt_cloud_task import ( + DbtCloudTask, + DbtCloudTaskParam, +) from databricks.bundles.jobs._models.dbt_task import DbtTask, DbtTaskParam from databricks.bundles.jobs._models.for_each_task import ( ForEachTask, @@ -121,6 +125,13 @@ class Task: The task refreshes a dashboard and sends a snapshot to subscribers. """ + dbt_cloud_task: VariableOrOptional[DbtCloudTask] = None + """ + :meta private: [EXPERIMENTAL] + + Task type for dbt cloud + """ + dbt_task: VariableOrOptional[DbtTask] = None """ The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. @@ -319,6 +330,13 @@ class TaskDict(TypedDict, total=False): The task refreshes a dashboard and sends a snapshot to subscribers. """ + dbt_cloud_task: VariableOrOptional[DbtCloudTaskParam] + """ + :meta private: [EXPERIMENTAL] + + Task type for dbt cloud + """ + dbt_task: VariableOrOptional[DbtTaskParam] """ The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. diff --git a/experimental/python/databricks/bundles/pipelines/_models/ingestion_config.py b/experimental/python/databricks/bundles/pipelines/_models/ingestion_config.py index 988227c43e..c452222df9 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/ingestion_config.py +++ b/experimental/python/databricks/bundles/pipelines/_models/ingestion_config.py @@ -4,10 +4,7 @@ from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value from databricks.bundles.core._variable import VariableOrOptional -from databricks.bundles.pipelines._models.report_spec import ( - ReportSpec, - ReportSpecParam, -) +from databricks.bundles.pipelines._models.report_spec import ReportSpec, ReportSpecParam from databricks.bundles.pipelines._models.schema_spec import SchemaSpec, SchemaSpecParam from databricks.bundles.pipelines._models.table_spec import TableSpec, TableSpecParam diff --git a/experimental/python/databricks/bundles/pipelines/_models/ingestion_source_type.py b/experimental/python/databricks/bundles/pipelines/_models/ingestion_source_type.py index 50754bee6a..b5ed997cfd 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/ingestion_source_type.py +++ b/experimental/python/databricks/bundles/pipelines/_models/ingestion_source_type.py @@ -13,6 +13,7 @@ class IngestionSourceType(Enum): SERVICENOW = "SERVICENOW" MANAGED_POSTGRESQL = "MANAGED_POSTGRESQL" ORACLE = "ORACLE" + TERADATA = "TERADATA" SHAREPOINT = "SHAREPOINT" DYNAMICS365 = "DYNAMICS365" @@ -29,6 +30,7 @@ class IngestionSourceType(Enum): "SERVICENOW", "MANAGED_POSTGRESQL", "ORACLE", + "TERADATA", "SHAREPOINT", "DYNAMICS365", ] diff --git a/experimental/python/databricks/bundles/pipelines/_models/pipeline.py b/experimental/python/databricks/bundles/pipelines/_models/pipeline.py index 8bf25fd1f1..936842ea92 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/experimental/python/databricks/bundles/pipelines/_models/pipeline.py @@ -182,6 +182,13 @@ class Pipeline(Resource): DBFS root directory for storing checkpoints and tables. """ + tags: VariableOrDict[str] = field(default_factory=dict) + """ + A map of tags associated with the pipeline. + These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. + A maximum of 25 tags can be added to the pipeline. + """ + target: VariableOrOptional[str] = None """ Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. @@ -325,6 +332,13 @@ class PipelineDict(TypedDict, total=False): DBFS root directory for storing checkpoints and tables. """ + tags: VariableOrDict[str] + """ + A map of tags associated with the pipeline. + These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. + A maximum of 25 tags can be added to the pipeline. + """ + target: VariableOrOptional[str] """ Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. diff --git a/experimental/python/databricks/bundles/version.py b/experimental/python/databricks/bundles/version.py index cdb90b1579..dd46d7456c 100644 --- a/experimental/python/databricks/bundles/version.py +++ b/experimental/python/databricks/bundles/version.py @@ -1 +1 @@ -__version__ = "0.254.0" +__version__ = "0.255.0" diff --git a/experimental/python/pyproject.toml b/experimental/python/pyproject.toml index bb6f60a561..31b832b135 100644 --- a/experimental/python/pyproject.toml +++ b/experimental/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "databricks-bundles" description = "Python support for Databricks Asset Bundles" -version = "0.254.0" +version = "0.255.0" authors = [ { name = "Gleb Kanterov", email = "gleb.kanterov@databricks.com" }, diff --git a/experimental/python/uv.lock b/experimental/python/uv.lock index 618b6be914..582e8de8ed 100644 --- a/experimental/python/uv.lock +++ b/experimental/python/uv.lock @@ -166,7 +166,7 @@ toml = [ [[package]] name = "databricks-bundles" -version = "0.254.0" +version = "0.255.0" source = { editable = "." } [package.dev-dependencies] diff --git a/go.mod b/go.mod index 4d304427d3..bc76833833 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/BurntSushi/toml v1.5.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.71.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.72.0 // Apache 2.0 github.com/fatih/color v1.18.0 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/gorilla/mux v1.8.1 // BSD 3-Clause @@ -46,8 +46,9 @@ require ( github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect + github.com/bmatcuk/doublestar/v4 v4.7.1 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect - github.com/cloudflare/circl v1.6.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -56,15 +57,18 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/yamlfmt v0.17.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/zclconf/go-cty v1.16.2 // indirect @@ -85,4 +89,7 @@ require ( gotest.tools/gotestsum v1.12.1 // indirect ) -tool gotest.tools/gotestsum +tool ( + github.com/google/yamlfmt/cmd/yamlfmt + gotest.tools/gotestsum +) diff --git a/go.sum b/go.sum index e1ccc0bf14..b4f563aa45 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= +github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q= +github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Ptu650= github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -30,14 +32,14 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= -github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/databricks/databricks-sdk-go v0.71.0 h1:YVNcvQUcgzlKesxDolDXSQPbNcCldubYLvM71hzVmUY= -github.com/databricks/databricks-sdk-go v0.71.0/go.mod h1:xBtjeP9nq+6MgTewZW1EcbRkD7aDY9gZvcRPcwPhZjw= +github.com/databricks/databricks-sdk-go v0.72.0 h1:vNS4zlpvNYiXsy/7/lzV7cuu/yOcT/1xpfuJw3+W3TA= +github.com/databricks/databricks-sdk-go v0.72.0/go.mod h1:xBtjeP9nq+6MgTewZW1EcbRkD7aDY9gZvcRPcwPhZjw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -101,6 +103,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/yamlfmt v0.17.0 h1:/tdp01rIlvLz3LgJ2NtMLnqgAadZm33P7GcPU680b+w= +github.com/google/yamlfmt v0.17.0/go.mod h1:gs0UEklJOYkUJ+OOCG0hg9n+DzucKDPlJElTUasVNK8= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= @@ -146,6 +150,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nwidger/jsoncolor v0.3.2 h1:rVJJlwAWDJShnbTYOQ5RM7yTA20INyKXlJ/fg4JMhHQ= github.com/nwidger/jsoncolor v0.3.2/go.mod h1:Cs34umxLbJvgBMnVNVqhji9BhoT/N/KinHqZptQ7cf4= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= diff --git a/integration/bundle/bundles/clusters/databricks_template_schema.json b/integration/bundle/bundles/clusters/databricks_template_schema.json deleted file mode 100644 index c1c5cf12eb..0000000000 --- a/integration/bundle/bundles/clusters/databricks_template_schema.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "properties": { - "unique_id": { - "type": "string", - "description": "Unique ID for job name" - }, - "spark_version": { - "type": "string", - "description": "Spark version used for job cluster" - }, - "node_type_id": { - "type": "string", - "description": "Node type id for job cluster" - } - } -} diff --git a/integration/bundle/clusters_test.go b/integration/bundle/clusters_test.go deleted file mode 100644 index b94b8365ef..0000000000 --- a/integration/bundle/clusters_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package bundle_test - -import ( - "testing" - - "github.com/databricks/cli/integration/internal/acc" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/google/uuid" - "github.com/stretchr/testify/require" -) - -func TestDeployBundleWithCluster(t *testing.T) { - if testutil.GetCloud(t) == testutil.AWS { - t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters") - } - - ctx, wt := acc.WorkspaceTest(t) - - nodeTypeId := testutil.GetCloud(t).NodeTypeID() - uniqueId := uuid.New().String() - root := initTestTemplate(t, ctx, "clusters", map[string]any{ - "unique_id": uniqueId, - "node_type_id": nodeTypeId, - "spark_version": defaultSparkVersion, - }) - - t.Cleanup(func() { - destroyBundle(t, ctx, root) - - cluster, err := wt.W.Clusters.GetByClusterName(ctx, "test-cluster-"+uniqueId) - if err != nil { - require.ErrorContains(t, err, "does not exist") - } else { - require.Contains(t, []compute.State{compute.StateTerminated, compute.StateTerminating}, cluster.State) - } - }) - - deployBundle(t, ctx, root) - - // Cluster should exists after bundle deployment - cluster, err := wt.W.Clusters.GetByClusterName(ctx, "test-cluster-"+uniqueId) - require.NoError(t, err) - require.NotNil(t, cluster) - - if testing.Short() { - t.Log("Skip the job run in short mode") - return - } - - out, err := runResource(t, ctx, root, "foo") - require.NoError(t, err) - require.Contains(t, out, "Hello World!") -} diff --git a/integration/bundle/local_state_staleness_test.go b/integration/bundle/local_state_staleness_test.go deleted file mode 100644 index 3984815044..0000000000 --- a/integration/bundle/local_state_staleness_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package bundle_test - -import ( - "context" - "testing" - - "github.com/databricks/cli/integration/internal/acc" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/databricks-sdk-go/listing" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLocalStateStaleness(t *testing.T) { - ctx, wt := acc.WorkspaceTest(t) - w := wt.W - - // The approach for this test is as follows: - // 1) First deploy of bundle instance A - // 2) First deploy of bundle instance B - // 3) Second deploy of bundle instance A - // Because of deploy (2), the locally cached state of bundle instance A should be stale. - // Then for deploy (3), it must use the remote state over the stale local state. - - nodeTypeId := testutil.GetCloud(t).NodeTypeID() - uniqueId := uuid.New().String() - initialize := func() string { - root := initTestTemplate(t, ctx, "basic", map[string]any{ - "unique_id": uniqueId, - "node_type_id": nodeTypeId, - "spark_version": defaultSparkVersion, - }) - - t.Cleanup(func() { - destroyBundle(t, ctx, root) - }) - - return root - } - - var err error - - bundleA := initialize() - bundleB := initialize() - - // 1) Deploy bundle A - deployBundle(t, ctx, bundleA) - - // 2) Deploy bundle B - deployBundle(t, ctx, bundleB) - - // 3) Deploy bundle A again - deployBundle(t, ctx, bundleA) - - // Assert that there is only a single job in the workspace corresponding to this bundle. - iter := w.Jobs.List(context.Background(), jobs.ListJobsRequest{ - Name: "test-job-basic-" + uniqueId, - }) - jobs, err := listing.ToSlice(context.Background(), iter) - require.NoError(t, err) - assert.Len(t, jobs, 1) -} diff --git a/integration/cmd/secrets/secrets_test.go b/integration/cmd/secrets/secrets_test.go index 43ad54de28..52f0274215 100644 --- a/integration/cmd/secrets/secrets_test.go +++ b/integration/cmd/secrets/secrets_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "fmt" + "os" "testing" "github.com/databricks/cli/integration/internal/acc" @@ -65,6 +66,21 @@ func assertSecretBytesValue(t *acc.WorkspaceT, scope, key string, expected []byt } func TestSecretsPutSecretStringValue(tt *testing.T) { + // aws-prod-ucws sets CLOUD_ENV to "ucws" + if os.Getenv("CLOUD_ENV") == "ucws" { + /* + FAIL integration/cmd/secrets.TestSecretsPutSecretStringValue (re-run 2) (1201.25s) + secrets_test.go:73: args: secrets, put-secret, cli-acc-c4ea35e34b52466cbf9a090c431d6a0b, test-key, --string-value, test-value + with-newlines + secrets_test.go:40: + Error Trace: /home/runner/work/eng-dev-ecosystem/eng-dev-ecosystem/ext/cli/integration/internal/acc/workspace.go:75 + Error: Received unexpected error: + wait: timed out: Finding instances for new nodes, acquiring more instances if necessary + Messages: Unexpected error from EnsureClusterIsRunning for clusterID=*** + */ + tt.Skip("Skipping to unblock PRs; re-enable if works") + } + ctx, t := acc.WorkspaceTest(tt) scope := temporarySecretScope(ctx, t) key := "test-key" @@ -79,6 +95,10 @@ func TestSecretsPutSecretStringValue(tt *testing.T) { } func TestSecretsPutSecretBytesValue(tt *testing.T) { + if os.Getenv("CLOUD_ENV") == "ucws" { + tt.Skip("Skipping to unblock PRs; re-enable if works") + } + ctx, t := acc.WorkspaceTest(tt) scope := temporarySecretScope(ctx, t) key := "test-key" diff --git a/libs/auth/arguments.go b/libs/auth/arguments.go index 17957ec511..899fc19a9e 100644 --- a/libs/auth/arguments.go +++ b/libs/auth/arguments.go @@ -18,8 +18,9 @@ func (a AuthArguments) ToOAuthArgument() (u2m.OAuthArgument, error) { Host: a.Host, AccountID: a.AccountID, } + host := cfg.CanonicalHostName() if cfg.IsAccountClient() { - return u2m.NewBasicAccountOAuthArgument(cfg.Host, cfg.AccountID) + return u2m.NewBasicAccountOAuthArgument(host, cfg.AccountID) } - return u2m.NewBasicWorkspaceOAuthArgument(cfg.Host) + return u2m.NewBasicWorkspaceOAuthArgument(host) } diff --git a/libs/auth/arguments_test.go b/libs/auth/arguments_test.go new file mode 100644 index 0000000000..d75827a771 --- /dev/null +++ b/libs/auth/arguments_test.go @@ -0,0 +1,84 @@ +package auth + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/credentials/u2m" + "github.com/stretchr/testify/assert" +) + +func TestToOAuthArgument(t *testing.T) { + tests := []struct { + name string + args AuthArguments + wantHost string + wantError bool + }{ + { + name: "workspace with no scheme", + args: AuthArguments{ + Host: "my-workspace.cloud.databricks.com", + }, + wantHost: "https://my-workspace.cloud.databricks.com", + }, + { + name: "workspace with https", + args: AuthArguments{ + Host: "https://my-workspace.cloud.databricks.com", + }, + wantHost: "https://my-workspace.cloud.databricks.com", + }, + { + name: "account with no scheme", + args: AuthArguments{ + Host: "accounts.cloud.databricks.com", + AccountID: "123456789", + }, + wantHost: "https://accounts.cloud.databricks.com", + }, + { + name: "account with https", + args: AuthArguments{ + Host: "https://accounts.cloud.databricks.com", + AccountID: "123456789", + }, + wantHost: "https://accounts.cloud.databricks.com", + }, + { + name: "workspace with query parameter", + args: AuthArguments{ + Host: "https://my-workspace.cloud.databricks.com?o=123456789", + }, + wantHost: "https://my-workspace.cloud.databricks.com", + }, + { + name: "workspace with query parameter and path", + args: AuthArguments{ + Host: "https://my-workspace.cloud.databricks.com/path?o=123456789", + }, + wantHost: "https://my-workspace.cloud.databricks.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.ToOAuthArgument() + if tt.wantError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + + // Check if we got the right type of argument and verify the hostname + if tt.args.AccountID != "" { + arg, ok := got.(u2m.AccountOAuthArgument) + assert.True(t, ok, "expected AccountOAuthArgument for account ID") + assert.Equal(t, tt.wantHost, arg.GetAccountHost()) + } else { + arg, ok := got.(u2m.WorkspaceOAuthArgument) + assert.True(t, ok, "expected WorkspaceOAuthArgument for workspace") + assert.Equal(t, tt.wantHost, arg.GetWorkspaceHost()) + } + }) + } +} diff --git a/libs/dyn/pattern.go b/libs/dyn/pattern.go index 6c20e9c9fb..efbe499761 100644 --- a/libs/dyn/pattern.go +++ b/libs/dyn/pattern.go @@ -1,6 +1,7 @@ package dyn import ( + "errors" "fmt" "slices" ) @@ -66,11 +67,39 @@ func AnyKey() patternComponent { return anyKeyComponent{} } +type expectedMapError struct { + p Path + v Value +} + +func (e expectedMapError) Error() string { + return fmt.Sprintf("expected a map at %q, found %s", e.p, e.v.Kind()) +} + +func IsExpectedMapError(err error) bool { + var target expectedMapError + return errors.As(err, &target) +} + +type expectedSequenceError struct { + p Path + v Value +} + +func (e expectedSequenceError) Error() string { + return fmt.Sprintf("expected a sequence at %q, found %s", e.p, e.v.Kind()) +} + +func IsExpectedSequenceError(err error) bool { + var target expectedSequenceError + return errors.As(err, &target) +} + // This function implements the patternComponent interface. func (c anyKeyComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { m, ok := v.AsMap() if !ok { - return InvalidValue, fmt.Errorf("expected a map at %q, found %s", prefix, v.Kind()) + return InvalidValue, expectedMapError{p: prefix, v: v} } m = m.Clone() @@ -105,7 +134,7 @@ func AnyIndex() patternComponent { func (c anyIndexComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { s, ok := v.AsSequence() if !ok { - return InvalidValue, fmt.Errorf("expected a sequence at %q, found %s", prefix, v.Kind()) + return InvalidValue, expectedSequenceError{p: prefix, v: v} } s = slices.Clone(s) diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go index 7d3ff36abe..80cec964ab 100644 --- a/libs/dyn/visit.go +++ b/libs/dyn/visit.go @@ -54,6 +54,34 @@ func IsIndexOutOfBoundsError(err error) bool { return errors.As(err, &target) } +type expectedMapToIndexError struct { + p Path + v Value +} + +func (e expectedMapToIndexError) Error() string { + return fmt.Sprintf("expected a map to index %q, found %s", e.p, e.v.Kind()) +} + +func IsExpectedMapToIndexError(err error) bool { + var target expectedMapToIndexError + return errors.As(err, &target) +} + +type expectedSequenceToIndexError struct { + p Path + v Value +} + +func (e expectedSequenceToIndexError) Error() string { + return fmt.Sprintf("expected a sequence to index %q, found %s", e.p, e.v.Kind()) +} + +func IsExpectedSequenceToIndexError(err error) bool { + var target expectedSequenceToIndexError + return errors.As(err, &target) +} + type visitOptions struct { // The function to apply to the value once found. // @@ -98,7 +126,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts case KindNil: return InvalidValue, cannotTraverseNilError{path} default: - return InvalidValue, fmt.Errorf("expected a map to index %q, found %s", path, v.Kind()) + return InvalidValue, expectedMapToIndexError{p: path, v: v} } m := v.MustMap() @@ -137,7 +165,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts case KindNil: return InvalidValue, cannotTraverseNilError{path} default: - return InvalidValue, fmt.Errorf("expected a sequence to index %q, found %s", path, v.Kind()) + return InvalidValue, expectedSequenceToIndexError{p: path, v: v} } s := v.MustSequence() diff --git a/libs/exec/exec.go b/libs/exec/exec.go index f3e777a423..4bec8b21b4 100644 --- a/libs/exec/exec.go +++ b/libs/exec/exec.go @@ -42,7 +42,7 @@ type command struct { func (c *command) Wait() error { // After the command has finished (cmd.Wait call), remove the temporary script file - defer os.Remove(c.execContext.scriptFile) + defer c.execContext.cleanup() err := c.cmd.Wait() if err != nil { @@ -140,7 +140,7 @@ func (e *Executor) Exec(ctx context.Context, command string) ([]byte, error) { if err != nil { return nil, err } - defer os.Remove(ec.scriptFile) + defer ec.cleanup() return cmd.CombinedOutput() } diff --git a/libs/exec/exec_test.go b/libs/exec/exec_test.go index 49ab2eba66..da4fe37f80 100644 --- a/libs/exec/exec_test.go +++ b/libs/exec/exec_test.go @@ -117,7 +117,11 @@ func TestExecutorNoShellFound(t *testing.T) { } func TestExecutorCleanupsTempFiles(t *testing.T) { - executor, err := NewCommandExecutor(".") + if runtime.GOOS != "windows" { + t.Skipf("cmd.exe is not available on non-Windows systems") + } + + executor, err := NewCommandExecutorWithExecutable(".", CmdExecutable) assert.NoError(t, err) cmd, ec, err := executor.prepareCommand(context.Background(), "echo 'Hello'") @@ -126,7 +130,8 @@ func TestExecutorCleanupsTempFiles(t *testing.T) { command, err := executor.start(cmd, ec) assert.NoError(t, err) - fileName := ec.args[1] + fileName := ec.scriptFile + assert.NotEmpty(t, fileName) assert.FileExists(t, fileName) err = command.Wait() diff --git a/libs/exec/shell.go b/libs/exec/shell.go index ee29eac8a0..77c690467b 100644 --- a/libs/exec/shell.go +++ b/libs/exec/shell.go @@ -14,9 +14,19 @@ type shell interface { type execContext struct { executable string args []string + + // scriptFile is the file that contains the command to be executed. + // We only use it for the cmd.exe shell since cmd.exe does not support + // inlining scripts. scriptFile string } +func (e *execContext) cleanup() { + if e.scriptFile != "" { + os.Remove(e.scriptFile) + } +} + func findShell() (shell, error) { for _, fn := range []func() (shell, error){ newBashShell, diff --git a/libs/exec/shell_bash.go b/libs/exec/shell_bash.go index 9f6b508f4d..b030887a3e 100644 --- a/libs/exec/shell_bash.go +++ b/libs/exec/shell_bash.go @@ -11,15 +11,9 @@ type bashShell struct { } func (s bashShell) prepare(command string) (*execContext, error) { - filename, err := createTempScript(command, ".sh") - if err != nil { - return nil, err - } - return &execContext{ executable: s.executable, - args: []string{"-e", filename}, - scriptFile: filename, + args: []string{"-c", command}, }, nil } diff --git a/libs/exec/shell_execv_test.go b/libs/exec/shell_execv_test.go index 580053b46c..1e76f74dc4 100644 --- a/libs/exec/shell_execv_test.go +++ b/libs/exec/shell_execv_test.go @@ -4,21 +4,20 @@ import ( "os/exec" "testing" - "github.com/databricks/cli/internal/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestShellExecvOpts(t *testing.T) { - newOpts, err := shellExecvOpts("echo hello", "/a/b/c", []string{"key1=value1", "key2=value2"}) + opts, err := shellExecvOpts("echo hello", "/a/b/c", []string{"key1=value1", "key2=value2"}) require.NoError(t, err) - assert.Equal(t, []string{"key1=value1", "key2=value2"}, newOpts.Env) - assert.Equal(t, "/a/b/c", newOpts.Dir) + assert.Equal(t, []string{"key1=value1", "key2=value2"}, opts.Env) + assert.Equal(t, "/a/b/c", opts.Dir) bashPath, err := exec.LookPath("bash") require.NoError(t, err) - assert.Equal(t, bashPath, newOpts.Args[0]) - assert.Equal(t, "-e", newOpts.Args[1]) - assert.Equal(t, "echo hello", testutil.ReadFile(t, newOpts.Args[2])) + assert.Equal(t, bashPath, opts.Args[0]) + assert.Equal(t, "-c", opts.Args[1]) + assert.Equal(t, "echo hello", opts.Args[2]) } diff --git a/libs/exec/shell_sh.go b/libs/exec/shell_sh.go index ca42ecfa46..a09d8ed7d3 100644 --- a/libs/exec/shell_sh.go +++ b/libs/exec/shell_sh.go @@ -10,15 +10,9 @@ type shShell struct { } func (s shShell) prepare(command string) (*execContext, error) { - filename, err := createTempScript(command, ".sh") - if err != nil { - return nil, err - } - return &execContext{ executable: s.executable, - args: []string{"-e", filename}, - scriptFile: filename, + args: []string{"-c", command}, }, nil } diff --git a/libs/patchwheel/filter.go b/libs/patchwheel/filter.go new file mode 100644 index 0000000000..f792b14ef3 --- /dev/null +++ b/libs/patchwheel/filter.go @@ -0,0 +1,57 @@ +package patchwheel + +import ( + "context" + + "github.com/databricks/cli/libs/log" +) + +// FilterLatestWheels iterates over provided wheel file paths, groups them by distribution name +// and, for every group, keeps only the wheel that has the latest version according to a best-effort +// comparison of the version strings. Returned slice preserves the order of the input slice – the +// first occurrence of the chosen wheel for every distribution is retained. +// +// The comparison is *heuristic*: the algorithm tokenises version strings into alternating numeric +// and non-numeric chunks. Numeric chunks are compared as integers, while non-numeric chunks are +// compared lexicographically. This covers common cases such as "1.2.10" > "1.2.3" and timestamps +// added via calculateNewVersion (e.g. "1.2.3+1741091696…" > "1.2.3"). It does not attempt to +// implement the full PEP 440 specification, which is unnecessary for the dynamic versions +// produced by this package. +func FilterLatestWheels(ctx context.Context, paths []string) []string { + // Build output incrementally, preserving the order of the *chosen* wheels. + out := make([]string, 0, len(paths)) + + // distribution -> index in out slice + bestIdx := make(map[string]int) + + for _, p := range paths { + info, err := ParseWheelFilename(p) + if err != nil { + // Unparsable: always keep. + out = append(out, p) + continue + } + + if idx, seen := bestIdx[info.Distribution]; !seen { + // First wheel for this distribution. + bestIdx[info.Distribution] = len(out) + out = append(out, p) + continue + } else { + // Compare against the current winner. + winnerPath := out[idx] + winnerInfo, _ := ParseWheelFilename(winnerPath) // guaranteed parseable + + if compareVersion(info.Version, winnerInfo.Version) > 0 { + // Current wheel wins: replace earlier entry in-place. + log.Debugf(ctx, "Skipping wheel %s (older than %s)", winnerPath, p) + out[idx] = p + } else { + // Current wheel loses. + log.Debugf(ctx, "Skipping wheel %s (older than %s)", p, winnerPath) + } + } + } + + return out +} diff --git a/libs/patchwheel/filter_test.go b/libs/patchwheel/filter_test.go new file mode 100644 index 0000000000..2ed2af831b --- /dev/null +++ b/libs/patchwheel/filter_test.go @@ -0,0 +1,33 @@ +package patchwheel + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFilterLatestWheels(t *testing.T) { + paths := []string{ + "project_name_bvs7tide6bhhpjy4dmcsb2qg44-0.0.1+20250604.74809-py3-none-any.whl", + "not-a-wheel.txt", + "mypkg-0.1.0-py3-none-any.whl", + "mypkg-0.2.0-py3-none-any.whl", + "other-1.0.0-py3-none-any.whl", + "other-0.9.0-py3-none-any.whl", + "project_name_bvs7tide6bhhpjy4dmcsb2qg44-0.0.1+20250604.74804-py3-none-any.whl", + "not-a-wheel.whl", + "hello-1.2.3-py3-none-any.whl", + "hello-1.2.3+1741091696780123321-py3-none-any.whl", + } + + filtered := FilterLatestWheels(context.Background(), paths) + require.ElementsMatch(t, []string{ + "project_name_bvs7tide6bhhpjy4dmcsb2qg44-0.0.1+20250604.74809-py3-none-any.whl", + "not-a-wheel.txt", + "mypkg-0.2.0-py3-none-any.whl", + "other-1.0.0-py3-none-any.whl", + "not-a-wheel.whl", + "hello-1.2.3+1741091696780123321-py3-none-any.whl", + }, filtered) +} diff --git a/libs/patchwheel/version.go b/libs/patchwheel/version.go new file mode 100644 index 0000000000..af5b49bf58 --- /dev/null +++ b/libs/patchwheel/version.go @@ -0,0 +1,79 @@ +package patchwheel + +import ( + "strconv" + "unicode" +) + +// compareVersion compares version strings a and b. +// Returns: +// +// 1 if a > b +// -1 if a < b +// 0 if equal. +// +// The algorithm splits each string into consecutive numeric and non-numeric tokens and compares them +// pair-wise: +// - Numeric tokens are compared as integers. +// - Non-numeric tokens are compared lexicographically. +// - Numeric tokens are considered greater than non-numeric tokens when types differ. +// +// Missing tokens are treated as zero-length strings / 0. +func compareVersion(a, b string) int { + ta := tokenizeVersion(a) + tb := tokenizeVersion(b) + + minLen := min(len(ta), len(tb)) + + for i, tokA := range ta[:minLen] { + tokB := tb[i] + + if tokA.numeric && tokB.numeric { + intA, _ := strconv.Atoi(tokA.value) + intB, _ := strconv.Atoi(tokB.value) + if intA != intB { + if intA > intB { + return 1 + } + return -1 + } + continue + } + + if tokA.value != tokB.value { + if tokA.value > tokB.value { + return 1 + } + return -1 + } + } + + // All shared tokens are equal; the longer version wins if it has extra tokens. + if len(ta) > len(tb) { + return 1 + } + if len(tb) > len(ta) { + return -1 + } + return 0 +} + +type versionToken struct { + numeric bool + value string +} + +func tokenizeVersion(v string) []versionToken { + var tokens []versionToken + start := 0 + for start < len(v) { + isDigit := unicode.IsDigit(rune(v[start])) + end := start + for end < len(v) && unicode.IsDigit(rune(v[end])) == isDigit { + end++ + } + tokens = append(tokens, versionToken{numeric: isDigit, value: v[start:end]}) + start = end + } + return tokens +} diff --git a/libs/patchwheel/version_test.go b/libs/patchwheel/version_test.go new file mode 100644 index 0000000000..2930004555 --- /dev/null +++ b/libs/patchwheel/version_test.go @@ -0,0 +1,37 @@ +package patchwheel + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCompareVersion(t *testing.T) { + cases := []struct { + a, b string + expect int + }{ + {"1.2.10", "1.2.3", 1}, + {"1.2.3", "1.2.3", 0}, + {"1.2.3+2", "1.2.3", 1}, + {"10.0.0", "2.0.0", 1}, + {"1.2.3a", "1.2.3", 1}, // non-numeric suffix greater lexicographically + {"1.2.3.1", "1.2.3", 1}, // leftover tokens make version greater + {"1.2.3", "1.2.3.0", -1}, + {"0.0.1+20250604.74804", "0.0.1+20250604.74809", -1}, + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("%s_vs_%s", tc.a, tc.b), func(t *testing.T) { + got := compareVersion(tc.a, tc.b) + require.Equal(t, tc.expect, got) + }) + + // Mirror case + t.Run(fmt.Sprintf("%s_vs_%s_mirror", tc.b, tc.a), func(t *testing.T) { + got := compareVersion(tc.b, tc.a) + require.Equal(t, -tc.expect, got) + }) + } +} diff --git a/libs/python/utils.go b/libs/python/utils.go deleted file mode 100644 index b94f5d4f65..0000000000 --- a/libs/python/utils.go +++ /dev/null @@ -1,48 +0,0 @@ -package python - -import ( - "context" - "os" - "path/filepath" - "strings" - - "github.com/databricks/cli/libs/log" -) - -func CleanupWheelFolder(dir string) { - // there or not there - we don't care - os.RemoveAll(filepath.Join(dir, "__pycache__")) - os.RemoveAll(filepath.Join(dir, "build")) - eggInfo := FindFilesWithSuffixInPath(dir, ".egg-info") - if len(eggInfo) == 0 { - return - } - for _, f := range eggInfo { - os.RemoveAll(f) - } -} - -func FindFilesWithSuffixInPath(dir, suffix string) []string { - f, err := os.Open(dir) - if err != nil { - log.Debugf(context.Background(), "open dir %s: %s", dir, err) - return nil - } - defer f.Close() - - entries, err := f.ReadDir(0) - if err != nil { - log.Debugf(context.Background(), "read dir %s: %s", dir, err) - // todo: log - return nil - } - - var files []string - for _, child := range entries { - if !strings.HasSuffix(child.Name(), suffix) { - continue - } - files = append(files, filepath.Join(dir, child.Name())) - } - return files -} diff --git a/libs/python/utils_test.go b/libs/python/utils_test.go deleted file mode 100644 index 1656d1ecb4..0000000000 --- a/libs/python/utils_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package python - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFindFilesWithSuffixInPath(t *testing.T) { - dir, err := os.Getwd() - require.NoError(t, err) - - files := FindFilesWithSuffixInPath(dir, "test.go") - - matches, err := filepath.Glob(filepath.Join(dir, "*test.go")) - require.NoError(t, err) - - require.ElementsMatch(t, files, matches) -} diff --git a/libs/structdiff/diff.go b/libs/structdiff/diff.go index d22dbbb814..de388132c9 100644 --- a/libs/structdiff/diff.go +++ b/libs/structdiff/diff.go @@ -6,7 +6,6 @@ import ( "slices" "sort" - "github.com/databricks/cli/libs/structdiff/jsontag" "github.com/databricks/cli/libs/structdiff/structpath" ) @@ -123,8 +122,7 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan continue } - tag := jsontag.JSONTag(sf.Tag.Get("json")) - node := structpath.NewStructField(path, tag, sf.Name) + node := structpath.NewStructField(path, sf.Tag, sf.Name) v1Field := s1.Field(i) v2Field := s2.Field(i) @@ -132,7 +130,7 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan zero2 := v2Field.IsZero() if zero1 || zero2 { - if tag.OmitEmpty() { + if node.JSONTag().OmitEmpty() { if zero1 { if !slices.Contains(forced1, sf.Name) { v1Field = reflect.ValueOf(nil) diff --git a/libs/structdiff/structpath/path.go b/libs/structdiff/structpath/path.go index b6fb55a84a..29f3b59577 100644 --- a/libs/structdiff/structpath/path.go +++ b/libs/structdiff/structpath/path.go @@ -2,9 +2,10 @@ package structpath import ( "fmt" + "reflect" "strconv" - "github.com/databricks/cli/libs/structdiff/jsontag" + "github.com/databricks/cli/libs/structdiff/structtag" ) const ( @@ -18,18 +19,23 @@ const ( // PathNode represents a node in a path for struct diffing. // It can represent struct fields, map keys, or array/slice indices. type PathNode struct { - prev *PathNode - jsonTag jsontag.JSONTag // For lazy JSON key resolution - key string // Computed key (JSON key for structs, string key for maps, or Go field name for fallback) + prev *PathNode + jsonTag structtag.JSONTag // For lazy JSON key resolution + bundleTag structtag.BundleTag + key string // Computed key (JSON key for structs, string key for maps, or Go field name for fallback) // If index >= 0, the node specifies a slice/array index in index. // If index < 0, this describes the type of node (see tagStruct and other consts above) index int } -func (p *PathNode) JSONTag() jsontag.JSONTag { +func (p *PathNode) JSONTag() structtag.JSONTag { return p.jsonTag } +func (p *PathNode) BundleTag() structtag.BundleTag { + return p.bundleTag +} + func (p *PathNode) IsRoot() bool { return p == nil } @@ -113,12 +119,16 @@ func NewMapKey(prev *PathNode, key string) *PathNode { // NewStructField creates a new PathNode for a struct field. // The jsonTag is used for lazy JSON key resolution, and fieldName is used as fallback. -func NewStructField(prev *PathNode, jsonTag jsontag.JSONTag, fieldName string) *PathNode { +func NewStructField(prev *PathNode, tag reflect.StructTag, fieldName string) *PathNode { + jsonTag := structtag.JSONTag(tag.Get("json")) + bundleTag := structtag.BundleTag(tag.Get("bundle")) + return &PathNode{ - prev: prev, - jsonTag: jsonTag, - key: fieldName, - index: tagUnresolvedStruct, + prev: prev, + jsonTag: jsonTag, + bundleTag: bundleTag, + key: fieldName, + index: tagUnresolvedStruct, } } diff --git a/libs/structdiff/structpath/path_test.go b/libs/structdiff/structpath/path_test.go index 7a575835fc..73815b8dfb 100644 --- a/libs/structdiff/structpath/path_test.go +++ b/libs/structdiff/structpath/path_test.go @@ -1,9 +1,9 @@ package structpath import ( + "reflect" "testing" - "github.com/databricks/cli/libs/structdiff/jsontag" "github.com/stretchr/testify/assert" ) @@ -42,28 +42,28 @@ func TestPathNode(t *testing.T) { }, { name: "struct field with JSON tag", - node: NewStructField(nil, jsontag.JSONTag("json_name"), "GoFieldName"), + node: NewStructField(nil, reflect.StructTag(`json:"json_name"`), "GoFieldName"), String: ".json_name", DynPath: "json_name", Field: "json_name", }, { name: "struct field without JSON tag (fallback to Go name)", - node: NewStructField(nil, jsontag.JSONTag(""), "GoFieldName"), + node: NewStructField(nil, reflect.StructTag(""), "GoFieldName"), String: ".GoFieldName", DynPath: "GoFieldName", Field: "GoFieldName", }, { name: "struct field with dash JSON tag", - node: NewStructField(nil, jsontag.JSONTag("-"), "GoFieldName"), + node: NewStructField(nil, reflect.StructTag(`json:"-"`), "GoFieldName"), String: ".-", DynPath: "-", Field: "-", }, { name: "struct field with JSON tag options", - node: NewStructField(nil, jsontag.JSONTag("lazy_field,omitempty"), "LazyField"), + node: NewStructField(nil, reflect.StructTag(`json:"lazy_field,omitempty"`), "LazyField"), String: ".lazy_field", DynPath: "lazy_field", Field: "lazy_field", @@ -85,21 +85,21 @@ func TestPathNode(t *testing.T) { // Two node tests { name: "struct field -> array index", - node: NewIndex(NewStructField(nil, jsontag.JSONTag("items"), "Items"), 3), + node: NewIndex(NewStructField(nil, reflect.StructTag(`json:"items"`), "Items"), 3), String: ".items[3]", DynPath: "items[3]", Index: 3, }, { name: "struct field -> map key", - node: NewMapKey(NewStructField(nil, jsontag.JSONTag("config"), "Config"), "database"), + node: NewMapKey(NewStructField(nil, reflect.StructTag(`json:"config"`), "Config"), "database"), String: `.config["database"]`, DynPath: "config.database", MapKey: "database", }, { name: "struct field -> struct field", - node: NewStructField(NewStructField(nil, jsontag.JSONTag("user"), "User"), jsontag.JSONTag("name"), "Name"), + node: NewStructField(NewStructField(nil, reflect.StructTag(`json:"user"`), "User"), reflect.StructTag(`json:"name"`), "Name"), String: ".user.name", DynPath: "user.name", Field: "name", @@ -113,14 +113,14 @@ func TestPathNode(t *testing.T) { }, { name: "map key -> struct field", - node: NewStructField(NewMapKey(nil, "primary"), jsontag.JSONTag("host"), "Host"), + node: NewStructField(NewMapKey(nil, "primary"), reflect.StructTag(`json:"host"`), "Host"), String: `["primary"].host`, DynPath: `primary.host`, Field: "host", }, { name: "array index -> struct field", - node: NewStructField(NewIndex(nil, 2), jsontag.JSONTag("id"), "ID"), + node: NewStructField(NewIndex(nil, 2), reflect.StructTag(`json:"id"`), "ID"), String: "[2].id", Field: "id", }, @@ -133,21 +133,21 @@ func TestPathNode(t *testing.T) { }, { name: "struct field without JSON tag -> struct field with JSON tag", - node: NewStructField(NewStructField(nil, jsontag.JSONTag(""), "Parent"), jsontag.JSONTag("child_name"), "ChildName"), + node: NewStructField(NewStructField(nil, reflect.StructTag(""), "Parent"), reflect.StructTag(`json:"child_name"`), "ChildName"), String: ".Parent.child_name", DynPath: "Parent.child_name", Field: "child_name", }, { name: "any key", - node: NewAnyKey(NewStructField(nil, jsontag.JSONTag(""), "Parent")), + node: NewAnyKey(NewStructField(nil, reflect.StructTag(""), "Parent")), String: ".Parent[*]", DynPath: "Parent.*", AnyKey: true, }, { name: "any index", - node: NewAnyIndex(NewStructField(nil, jsontag.JSONTag(""), "Parent")), + node: NewAnyIndex(NewStructField(nil, reflect.StructTag(""), "Parent")), String: ".Parent[*]", DynPath: "Parent[*]", AnyIndex: true, diff --git a/libs/structdiff/structtag/bundletag.go b/libs/structdiff/structtag/bundletag.go new file mode 100644 index 0000000000..9b7bb2d0ac --- /dev/null +++ b/libs/structdiff/structtag/bundletag.go @@ -0,0 +1,13 @@ +package structtag + +// BundleTag represents a struct field's `bundle` tag as a string. +// It provides methods to extract information from the tag. +type BundleTag string + +func (tag BundleTag) ReadOnly() bool { + return hasOption(string(tag), "readonly") +} + +func (tag BundleTag) Internal() bool { + return hasOption(string(tag), "internal") +} diff --git a/libs/structdiff/structtag/bundletag_test.go b/libs/structdiff/structtag/bundletag_test.go new file mode 100644 index 0000000000..3d2129ec76 --- /dev/null +++ b/libs/structdiff/structtag/bundletag_test.go @@ -0,0 +1,37 @@ +package structtag + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBundleTagMethods(t *testing.T) { + tests := []struct { + tag string + isReadOnly bool + isInternal bool + }{ + // only one annotation. + {tag: "readonly", isReadOnly: true}, + {tag: "internal", isInternal: true}, + + // multiple annotations. + {tag: "readonly,internal", isReadOnly: true, isInternal: true}, + + // unknown annotations are ignored. + {tag: "something"}, + {tag: "-"}, + {tag: "name,string"}, + {tag: "weird,whatever,readonly,foo", isReadOnly: true}, + } + + for _, test := range tests { + t.Run(test.tag, func(t *testing.T) { + tag := BundleTag(test.tag) + + assert.Equal(t, test.isReadOnly, tag.ReadOnly()) + assert.Equal(t, test.isInternal, tag.Internal()) + }) + } +} diff --git a/libs/structdiff/jsontag/jsontag.go b/libs/structdiff/structtag/jsontag.go similarity index 82% rename from libs/structdiff/jsontag/jsontag.go rename to libs/structdiff/structtag/jsontag.go index 4483d7c864..904909a27f 100644 --- a/libs/structdiff/jsontag/jsontag.go +++ b/libs/structdiff/structtag/jsontag.go @@ -1,4 +1,4 @@ -package jsontag +package structtag import "strings" @@ -45,13 +45,17 @@ func (tag JSONTag) hasOption(option string) bool { s = s[idx+1:] } + return hasOption(s, option) +} + +func hasOption(tag, option string) bool { // Walk the comma-separated options - for len(s) > 0 { - opt := s - if i := strings.IndexByte(s, ','); i != -1 { - opt, s = s[:i], s[i+1:] + for len(tag) > 0 { + opt := tag + if i := strings.IndexByte(tag, ','); i != -1 { + opt, tag = tag[:i], tag[i+1:] } else { - s = "" + tag = "" } if opt == option { diff --git a/libs/structdiff/jsontag/jsontag_test.go b/libs/structdiff/structtag/jsontag_test.go similarity index 98% rename from libs/structdiff/jsontag/jsontag_test.go rename to libs/structdiff/structtag/jsontag_test.go index c4d21223ae..b45fd566ea 100644 --- a/libs/structdiff/jsontag/jsontag_test.go +++ b/libs/structdiff/structtag/jsontag_test.go @@ -1,4 +1,4 @@ -package jsontag +package structtag import "testing" diff --git a/libs/structwalk/walk.go b/libs/structwalk/walk.go index 5f98556dec..b1b87b861a 100644 --- a/libs/structwalk/walk.go +++ b/libs/structwalk/walk.go @@ -6,7 +6,6 @@ import ( "slices" "sort" - "github.com/databricks/cli/libs/structdiff/jsontag" "github.com/databricks/cli/libs/structdiff/structpath" ) @@ -113,16 +112,15 @@ func walkStruct(path *structpath.PathNode, s reflect.Value, visit VisitFunc) { if sf.Name == "ForceSendFields" { continue } - tag := sf.Tag.Get("json") - if tag == "-" { + + node := structpath.NewStructField(path, sf.Tag, sf.Name) + if node.JSONTag().Name() == "-" { continue // skip fields without json name } - jsonTag := jsontag.JSONTag(tag) - fieldVal := s.Field(i) - node := structpath.NewStructField(path, jsonTag, sf.Name) + fieldVal := s.Field(i) // Skip zero values with omitempty unless field is explicitly forced. - if jsonTag.OmitEmpty() && fieldVal.IsZero() && !slices.Contains(forced, sf.Name) { + if node.JSONTag().OmitEmpty() && fieldVal.IsZero() && !slices.Contains(forced, sf.Name) { continue } diff --git a/libs/structwalk/walk_test.go b/libs/structwalk/walk_test.go index 8122915292..4967894457 100644 --- a/libs/structwalk/walk_test.go +++ b/libs/structwalk/walk_test.go @@ -83,3 +83,31 @@ func TestValueJobSettings(t *testing.T) { ".timeout_seconds": 3600, }, flatten(t, jobSettings)) } + +func TestValueBundleTag(t *testing.T) { + type Foo struct { + A string `bundle:"readonly"` + B string `bundle:"internal"` + C string + D string `bundle:"internal,readonly"` + } + + var readonly, internal []string + err := Walk(Foo{ + A: "a", + B: "b", + C: "c", + D: "d", + }, func(path *structpath.PathNode, value any) { + if path.BundleTag().ReadOnly() { + readonly = append(readonly, path.String()) + } + if path.BundleTag().Internal() { + internal = append(internal, path.String()) + } + }) + require.NoError(t, err) + + assert.Equal(t, []string{".A", ".D"}, readonly) + assert.Equal(t, []string{".B", ".D"}, internal) +} diff --git a/libs/structwalk/walktype.go b/libs/structwalk/walktype.go index 2c1252de39..f71cdb3797 100644 --- a/libs/structwalk/walktype.go +++ b/libs/structwalk/walktype.go @@ -4,31 +4,34 @@ import ( "errors" "reflect" - "github.com/databricks/cli/libs/structdiff/jsontag" "github.com/databricks/cli/libs/structdiff/structpath" ) -// VisitTypeFunc is invoked for every scalar (int, uint, float, string, bool) field type encountered while walking t. +// VisitTypeFunc is invoked for fields encountered while walking typ. This includes both leaf nodes as well as any +// intermediate nodes encountered while walking the struct tree. // // path PathNode representing the JSON-style path to the field. // typ the field's type – if the field is a pointer to a scalar the pointer type is preserved; // the callback receives the actual type (e.g., *string, *int, etc.). // +// The function returns a boolean: +// continueWalk: if true, the WalkType function will continue recursively walking the current field. +// if false, the WalkType function will skip walking the current field and all its children. +// // NOTE: Fields lacking a json tag or tagged as "-" are ignored entirely. -// Composite kinds (struct, slice/array, map, interface, function, chan, etc.) are *not* visited, but the walk -// traverses them to reach nested scalar field types (except interface & func). Only maps with string keys are -// traversed so that paths stay JSON-like. +// Dynamic types like func, chan, interface, etc. are *not* visited. +// Only maps with string keys are traversed so that paths stay JSON-like. // // The walk is depth-first and deterministic (map keys are sorted lexicographically). // // Example: -// err := structwalk.WalkType(reflect.TypeOf(cfg), func(path *structpath.PathNode, t reflect.Type) { -// fmt.Printf("%s = %v\n", path.String(), t) +// err := structwalk.WalkType(reflect.TypeOf(cfg), func(path *structpath.PathNode, typ reflect.Type) { +// fmt.Printf("%s = %v\n", path.String(), typ) // }) // // ****************************************************************************************************** -type VisitTypeFunc func(path *structpath.PathNode, typ reflect.Type) +type VisitTypeFunc func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) // WalkType validates that t is a struct or pointer to one and starts the recursive traversal. func WalkType(t reflect.Type, visit VisitTypeFunc) error { @@ -48,11 +51,22 @@ func walkTypeValue(path *structpath.PathNode, typ reflect.Type, visit VisitTypeF return } - kind := typ.Kind() + // Call visit on all nodes including the root node. We call visit before + // dereferencing pointers to ensure that the visit callback receives + // the actual type of the field. + continueWalk := visit(path, typ) + if !continueWalk { + return + } + + // Dereference pointers. + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + // Return early if we're at a leaf scalar. + kind := typ.Kind() if isScalar(kind) { - // Primitive scalar at the leaf – invoke. - visit(path, typ) return } @@ -64,9 +78,6 @@ func walkTypeValue(path *structpath.PathNode, typ reflect.Type, visit VisitTypeF visitedCount[typ]++ switch kind { - case reflect.Pointer: - walkTypeValue(path, typ.Elem(), visit, visitedCount) - case reflect.Struct: walkTypeStruct(path, typ, visit, visitedCount) @@ -93,25 +104,20 @@ func walkTypeStruct(path *structpath.PathNode, st reflect.Type, visit VisitTypeF if sf.PkgPath != "" { continue // unexported } - tag := sf.Tag.Get("json") + node := structpath.NewStructField(path, sf.Tag, sf.Name) // Handle embedded structs (anonymous fields without json tags) - if sf.Anonymous && tag == "" { + if sf.Anonymous && node.JSONTag() == "" { // For embedded structs, walk the embedded type at the current path level // This flattens the embedded struct's fields into the parent struct walkTypeValue(path, sf.Type, visit, visitedCount) continue } - if tag == "-" { - continue // skip fields without json name - } - jsonTag := jsontag.JSONTag(tag) - if jsonTag.Name() == "-" { + if node.JSONTag().Name() == "-" { continue } - fieldType := sf.Type - node := structpath.NewStructField(path, jsonTag, sf.Name) - walkTypeValue(node, fieldType, visit, visitedCount) + + walkTypeValue(node, sf.Type, visit, visitedCount) } } diff --git a/libs/structwalk/walktype_bench_test.go b/libs/structwalk/walktype_bench_test.go index 78ddaef374..f780260b8d 100644 --- a/libs/structwalk/walktype_bench_test.go +++ b/libs/structwalk/walktype_bench_test.go @@ -11,8 +11,9 @@ import ( func countFields(typ reflect.Type) (int, error) { fieldCount := 0 - err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type) { + err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) { fieldCount++ + return true }) return fieldCount, err } diff --git a/libs/structwalk/walktype_test.go b/libs/structwalk/walktype_test.go index fd20c8f663..54d43efa2a 100644 --- a/libs/structwalk/walktype_test.go +++ b/libs/structwalk/walktype_test.go @@ -11,10 +11,16 @@ import ( "github.com/stretchr/testify/require" ) -func getFields(t *testing.T, typ reflect.Type) map[string]any { +func getScalarFields(t *testing.T, typ reflect.Type) map[string]any { results := make(map[string]any) - err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type) { - results[path.String()] = reflect.Zero(typ).Interface() + err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) { + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if isScalar(typ.Kind()) { + results[path.String()] = reflect.Zero(typ).Interface() + } + return true }) require.NoError(t, err) return results @@ -27,11 +33,11 @@ func TestTypeNilCallback(t *testing.T) { } func TestTypeNil(t *testing.T) { - assert.Equal(t, map[string]any{}, getFields(t, reflect.TypeOf(nil))) + assert.Equal(t, map[string]any{}, getScalarFields(t, reflect.TypeOf(nil))) } func TestTypeScalar(t *testing.T) { - assert.Equal(t, map[string]any{"": 0}, getFields(t, reflect.TypeOf(5))) + assert.Equal(t, map[string]any{"": 0}, getScalarFields(t, reflect.TypeOf(5))) } func TestTypes(t *testing.T) { @@ -55,7 +61,7 @@ func TestTypes(t *testing.T) { ".omit_str": "", ".valid_field": "", ".valid_field_ptr": "", - }, getFields(t, reflect.TypeOf(Types{}))) + }, getScalarFields(t, reflect.TypeOf(Types{}))) } func TestTypeSelf(t *testing.T) { @@ -69,11 +75,11 @@ func TestTypeSelf(t *testing.T) { ".SelfReference.valid_field": "", ".SelfSlicePtr[*].valid_field": "", ".SelfSlice[*].valid_field": "", - }, getFields(t, reflect.TypeOf(Self{}))) + }, getScalarFields(t, reflect.TypeOf(Self{}))) } func testStruct(t *testing.T, typ reflect.Type, minLen, maxLen int, present map[string]any, notPresent []string) { - results := getFields(t, typ) + results := getScalarFields(t, typ) assert.Greater(t, len(results), minLen, "Expected to find many fields in %s", typ) assert.Less(t, len(results), maxLen, "Expected to find not so many fields in %s", typ) @@ -117,7 +123,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeOf(config.Root{}), - 3400, 3500, // 3487 at this time + 3500, 3600, // 3516 at this time map[string]any{ ".bundle.target": "", `.variables[*].lookup.dashboard`: "", @@ -145,3 +151,127 @@ func TestTypeRoot(t *testing.T) { nil, ) } + +func getReadonlyFields(t *testing.T, typ reflect.Type) []string { + var results []string + err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) { + if path == nil { + return true + } + if path.BundleTag().ReadOnly() { + results = append(results, path.DynPath()) + } + return true + }) + require.NoError(t, err) + return results +} + +func TestTypeReadonlyFields(t *testing.T) { + readonlyFields := getReadonlyFields(t, reflect.TypeOf(config.Root{})) + + expected := []string{ + "bundle.mode", + "bundle.target", + "resources.jobs.*.id", + "resources.pipelines.*.id", + "workspace.current_user.short_name", + } + + for _, v := range expected { + assert.Contains(t, readonlyFields, v) + } +} + +func TestTypeBundleTag(t *testing.T) { + type Foo struct { + A string `bundle:"readonly"` + B string `bundle:"internal"` + C string + D string `bundle:"internal,readonly"` + } + + var readonly, internal []string + err := WalkType(reflect.TypeOf(Foo{}), func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) { + if path == nil { + return true + } + if path.BundleTag().ReadOnly() { + readonly = append(readonly, path.String()) + } + if path.BundleTag().Internal() { + internal = append(internal, path.String()) + } + return true + }) + require.NoError(t, err) + + assert.Equal(t, []string{".A", ".D"}, readonly) + assert.Equal(t, []string{".B", ".D"}, internal) +} + +func TestWalkTypeVisited(t *testing.T) { + type Inner struct { + A int + B ***int + } + + type Outer struct { + Inner Inner + MapInner map[string]*Inner + SliceInner []Inner + + C string + D bool + } + + var visited []string + err := WalkType(reflect.TypeOf(Outer{}), func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) { + if path == nil { + return true + } + visited = append(visited, path.String()) + return true + }) + require.NoError(t, err) + + assert.Equal(t, []string{ + ".Inner", + ".Inner.A", + ".Inner.B", + ".MapInner", + ".MapInner[*]", + ".MapInner[*].A", + ".MapInner[*].B", + ".SliceInner", + ".SliceInner[*]", + ".SliceInner[*].A", + ".SliceInner[*].B", + ".C", + ".D", + }, visited) +} + +func TestWalkSkip(t *testing.T) { + type Outer struct { + A int + B int + + Inner struct { + C int + } + + D int + } + + var seen []string + err := WalkType(reflect.TypeOf(Outer{}), func(path *structpath.PathNode, typ reflect.Type) (continueWalk bool) { + if path == nil { + return true + } + seen = append(seen, path.String()) + return path.String() != ".Inner" + }) + require.NoError(t, err) + assert.Equal(t, []string{".A", ".B", ".Inner", ".D"}, seen) +} diff --git a/libs/template/template.go b/libs/template/template.go index 43569050e2..6ed97a83d4 100644 --- a/libs/template/template.go +++ b/libs/template/template.go @@ -27,6 +27,7 @@ const ( DefaultPython TemplateName = "default-python" DefaultSql TemplateName = "default-sql" LakeflowPipelines TemplateName = "lakeflow-pipelines" + DLT TemplateName = "dlt" DbtSql TemplateName = "dbt-sql" MlopsStacks TemplateName = "mlops-stacks" DefaultPydabs TemplateName = "default-pydabs" @@ -54,6 +55,13 @@ var databricksTemplates = []Template{ Reader: &builtinReader{name: string(LakeflowPipelines)}, Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: LakeflowPipelines}}, }, + { + name: DLT, + hidden: true, + description: "The DLT template for Delta Live Tables pipelines", + Reader: &builtinReader{name: string(DLT)}, + Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: DLT}}, + }, { name: DbtSql, description: "The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)", diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index d991c06ffe..3b452014a6 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -7,6 +7,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml # Deployment targets. # The default schema, catalog, etc. for dbt are defined in dbt_profiles/profiles.yml diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl index e96931e2de..948c3eca90 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl @@ -4,42 +4,42 @@ {{- end}} # This file defines dbt profiles for deployed dbt jobs. {{.project_name}}: - target: dev # default target - outputs: + target: dev # default target + outputs: - # Doing local development with the dbt CLI? - # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' - # (See README.md) + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) - # The default target when deployed with the Databricks CLI - # N.B. when you use dbt from the command line, it uses the profile from .dbt/profiles.yml - dev: - type: databricks - method: http - catalog: {{$catalog}} + # The default target when deployed with the Databricks CLI + # N.B. when you use dbt from the command line, it uses the profile from .dbt/profiles.yml + dev: + type: databricks + method: http + catalog: {{$catalog}} {{- if (regexp "^yes").MatchString .personal_schemas}} - schema: "{{"{{"}} var('dev_schema') {{"}}"}}" + schema: "{{"{{"}} var('dev_schema') {{"}}"}}" {{- else}} - schema: "{{.shared_schema}}" + schema: "{{.shared_schema}}" {{- end}} - http_path: {{.http_path}} + http_path: {{.http_path}} - # The workspace host / token are provided by Databricks - # see databricks.yml for the workspace host used for 'dev' - host: "{{"{{"}} env_var('DBT_HOST') {{"}}"}}" - token: "{{"{{"}} env_var('DBT_ACCESS_TOKEN') {{"}}"}}" + # The workspace host / token are provided by Databricks + # see databricks.yml for the workspace host used for 'dev' + host: "{{"{{"}} env_var('DBT_HOST') {{"}}"}}" + token: "{{"{{"}} env_var('DBT_ACCESS_TOKEN') {{"}}"}}" - # The production target when deployed with the Databricks CLI - prod: - type: databricks - method: http - catalog: {{$catalog}} - schema: {{.shared_schema}} + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: {{$catalog}} + schema: {{.shared_schema}} - http_path: {{.http_path}} + http_path: {{.http_path}} - # The workspace host / token are provided by Databricks - # see databricks.yml for the workspace host used for 'prod' - host: "{{"{{"}} env_var('DBT_HOST') {{"}}"}}" - token: "{{"{{"}} env_var('DBT_ACCESS_TOKEN') {{"}}"}}" + # The workspace host / token are provided by Databricks + # see databricks.yml for the workspace host used for 'prod' + host: "{{"{{"}} env_var('DBT_HOST') {{"}}"}}" + token: "{{"{{"}} env_var('DBT_ACCESS_TOKEN') {{"}}"}}" diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_project.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_project.yml.tmpl index 11fbf051e3..91fb1be1b5 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_project.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/dbt_project.yml.tmpl @@ -15,7 +15,7 @@ seed-paths: ["src/seeds"] macro-paths: ["src/macros"] snapshot-paths: ["src/snapshots"] -clean-targets: # directories to be removed by `dbt clean` +clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 9e3efadd42..5ca04580e9 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -15,27 +15,26 @@ resources: tasks: - task_key: dbt - dbt_task: project_directory: ../ # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml profiles_directory: dbt_profiles/ commands: {{- if (regexp "^yes").MatchString .personal_schemas}} - # The dbt commands to run (see also dbt_profiles/profiles.yml; dev_schema is used in the dev profile) - - 'dbt deps --target=${bundle.target}' - - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + # The dbt commands to run (see also dbt_profiles/profiles.yml; dev_schema is used in the dev profile) + - 'dbt deps --target=${bundle.target}' + - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' {{- else}} - # The dbt commands to run (see also the dev/prod profiles in dbt_profiles/profiles.yml) - - 'dbt deps --target=${bundle.target}' - - 'dbt seed --target=${bundle.target}' - - 'dbt run --target=${bundle.target}' + # The dbt commands to run (see also the dev/prod profiles in dbt_profiles/profiles.yml) + - 'dbt deps --target=${bundle.target}' + - 'dbt seed --target=${bundle.target}' + - 'dbt run --target=${bundle.target}' {{- end}} libraries: - - pypi: - package: dbt-databricks>=1.8.0,<2.0.0 + - pypi: + package: dbt-databricks>=1.8.0,<2.0.0 new_cluster: spark_version: {{template "latest_lts_dbr_version"}} @@ -43,7 +42,7 @@ resources: data_security_mode: SINGLE_USER num_workers: 0 spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl index 04d22a764e..3bbfd4c7f3 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml targets: dev: diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index bcb7662ba6..9dea88dede 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -49,7 +49,7 @@ resources: {{- else if (eq .include_notebook "yes" )}} depends_on: - task_key: notebook_task - {{end}} +{{end}} {{- if $with_serverless }} environment_key: default {{- else }} @@ -85,6 +85,6 @@ resources: node_type_id: {{smallest_node_type}} data_security_mode: SINGLE_USER autoscale: - min_workers: 1 - max_workers: 4 + min_workers: 1 + max_workers: 4 {{end -}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index 6acdf40e78..2d61f52bd2 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -6,6 +6,7 @@ bundle: include: - resources/*.yml + - resources/*/*.yml # Variable declarations. These variables are assigned in the dev/prod targets below. variables: diff --git a/libs/template/templates/dlt/README.md b/libs/template/templates/dlt/README.md new file mode 100644 index 0000000000..008a23aecd --- /dev/null +++ b/libs/template/templates/dlt/README.md @@ -0,0 +1,3 @@ +# Lakeflow Pipelines + +Default template for Lakeflow Declarative Pipelines diff --git a/libs/template/templates/dlt/databricks_template_schema.json b/libs/template/templates/dlt/databricks_template_schema.json new file mode 100644 index 0000000000..765e5f4fb0 --- /dev/null +++ b/libs/template/templates/dlt/databricks_template_schema.json @@ -0,0 +1,47 @@ +{ + "welcome_message": "\nWelcome to the template for DLT Pipelines!", + "properties": { + "project_name": { + "type": "string", + "default": "pipelines_project", + "description": "Please provide the following details to tailor the template to your preferences.\n\nUnique name for this project\nproject_name", + "order": 1, + "pattern": "^[a-z0-9_]+$", + "pattern_match_failure_message": "Name must consist of lower case letters, numbers, and underscores." + }, + "workspace_host": { + "type": "string", + "description": "\nWorkspace URL to deploy to:\nworkspace_host [example: mycompany.cloud.databricks.com]", + "order": 2, + "pattern": "^[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}/?.*$", + "pattern_match_failure_message": "Must be a valid Databricks workspace URL" + }, + "default_catalog": { + "type": "string", + "default": "{{default_catalog}}", + "pattern": "^\\w*$", + "pattern_match_failure_message": "Invalid catalog name.", + "description": "\nPlease provide a default catalog to use{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}\ndefault_catalog", + "order": 3 + }, + "default_schema": { + "type": "string", + "default": "", + "pattern": "^\\w*$", + "pattern_match_failure_message": "Invalid schema name.", + "description": "\nPlease provide a default schema to use. Leave this blank to use the current username (recommended for collaboration).\ndefault_schema", + "order": 4 + }, + "language": { + "type": "string", + "default": "python", + "description": "\nInitial language for this project:\nlanguage", + "enum": [ + "python", + "sql" + ], + "order": 5 + } + }, + "success_message": "\n\nYour new project has been created in the '{{.project_name}}' directory!\n\nRefer to the README.md file for \"getting started\" instructions!" +} diff --git a/libs/template/templates/dlt/library/variables.tmpl b/libs/template/templates/dlt/library/variables.tmpl new file mode 100644 index 0000000000..957adb42cc --- /dev/null +++ b/libs/template/templates/dlt/library/variables.tmpl @@ -0,0 +1,33 @@ +{{- define `pipeline_name` -}} + {{ .project_name }}_pipeline +{{- end }} + +{{- define `job_name` -}} + {{ .project_name }}_job +{{- end }} + +{{- define `static_dev_schema` -}} + {{- if (eq .default_schema "") -}} + {{ short_name }} + {{- else -}} + {{ .default_schema }} + {{- end}} +{{- end }} + + +{{- define `dev_schema` -}} + {{- if (eq .default_schema "") -}} + ${workspace.current_user.short_name} + {{- else -}} + {{ .default_schema }} + {{- end}} +{{- end }} + + +{{- define `prod_schema` -}} + {{- if (eq .default_schema "") -}} + default + {{- else -}} + {{ .default_schema }} + {{- end}} +{{- end }} diff --git a/libs/template/templates/dlt/template/__preamble.tmpl b/libs/template/templates/dlt/template/__preamble.tmpl new file mode 100644 index 0000000000..85955e7026 --- /dev/null +++ b/libs/template/templates/dlt/template/__preamble.tmpl @@ -0,0 +1,43 @@ +# Preamble + +This file only contains template directives; it is skipped for the actual output. + +{{skip "__preamble"}} + +{{$isSQL := eq .language "sql"}} +{{$isPython := eq .language "python"}} +{{$isScala := eq .language "scala"}} +{{$isR := eq .language "r"}} + +{{if $isSQL}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.scala"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.scala"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.r"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.r"}} +{{else if $isPython}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.scala"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.scala"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.r"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.r"}} +{{else if $isScala}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.r"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.r"}} +{{else if $isR}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.scala"}} + {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.scala"}} +{{end}} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/.gitignore.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/.gitignore.tmpl new file mode 100644 index 0000000000..f6a3b5ff93 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/.gitignore.tmpl @@ -0,0 +1,8 @@ +.databricks/ +build/ +dist/ +__pycache__/ +*.egg-info +.venv/ +**/explorations/** +**/!explorations/README.md diff --git a/libs/template/templates/dlt/template/{{.project_name}}/.vscode/__builtins__.pyi b/libs/template/templates/dlt/template/{{.project_name}}/.vscode/__builtins__.pyi new file mode 100644 index 0000000000..0edd5181bc --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/.vscode/__builtins__.pyi @@ -0,0 +1,3 @@ +# Typings for Pylance in Visual Studio Code +# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md +from databricks.sdk.runtime import * diff --git a/libs/template/templates/dlt/template/{{.project_name}}/.vscode/extensions.json b/libs/template/templates/dlt/template/{{.project_name}}/.vscode/extensions.json new file mode 100644 index 0000000000..5d15eba363 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "databricks.databricks", + "ms-python.vscode-pylance", + "redhat.vscode-yaml" + ] +} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/.vscode/settings.json.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/.vscode/settings.json.tmpl new file mode 100644 index 0000000000..6a87715ae2 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/.vscode/settings.json.tmpl @@ -0,0 +1,22 @@ +{ + "python.analysis.stubPath": ".vscode", + "databricks.python.envFile": "${workspaceFolder}/.env", + "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + {{- /* Unfortunately extraPaths doesn't support globs!! See: https://github.com/microsoft/pylance-release/issues/973 */}} + "python.analysis.extraPaths": ["resources/{{.project_name}}_pipeline"], + "files.exclude": { + "**/*.egg-info": true, + "**/__pycache__": true, + ".pytest_cache": true, + }, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + }, +} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/README.md.tmpl new file mode 100644 index 0000000000..837213a189 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/README.md.tmpl @@ -0,0 +1,41 @@ +# {{.project_name}} + +The '{{.project_name}}' project was generated by using the Lakeflow Pipelines template. + +## Setup + +1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +2. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks auth login + ``` + +3. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from + https://docs.databricks.com/dev-tools/vscode-ext.html. Or the PyCharm plugin from + https://www.databricks.com/blog/announcing-pycharm-integration-databricks. + + +## Deploying resources + +1. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + +2. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + +3. Use the "summary" comand to review everything that was deployed: + ``` + $ databricks bundle summary + ``` + +4. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` diff --git a/libs/template/templates/dlt/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/databricks.yml.tmpl new file mode 100644 index 0000000000..3445a56053 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/databricks.yml.tmpl @@ -0,0 +1,25 @@ +# This defines the {{.project_name}} project. +project: + name: {{.project_name}} + +include: + - ./*.yml + +targets: + dev: + mode: development + deploy_on_run: true + workspace: + host: {{workspace_host}} + variables: + catalog: {{default_catalog}} + schema: {{if (eq .default_schema "")}}${workspace.current_user.short_name} # the current username, e.g. {{short_name}}{{else}}{{.default_schema}}{{end}} + + prod: + mode: production + owner: {{user_name}} + workspace: + host: {{workspace_host}} + variables: + catalog: {{.default_catalog}} + schema: {{if (eq .default_schema "")}}default{{else}}{{.default_schema}}{{end}} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/README.md.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/README.md.tmpl new file mode 100644 index 0000000000..b085a301a6 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/README.md.tmpl @@ -0,0 +1,48 @@ +{{- if (eq .language "python") -}} + +# {{template `pipeline_name` .}} + +This folder defines all source code for the {{template `pipeline_name` .}} pipeline: + +- `explorations`: Ad-hoc notebooks used to explore the data processed by this pipeline. +- `transformations`: All dataset definitions and transformations. +- `utilities` (optional): Utility functions and Python modules used in this pipeline. +- `data_sources` (optional): View definitions describing the source data for this pipeline. + +## Getting Started + +To get started, go to the `transformations` folder -- most of the relevant source code lives there: + +* By convention, every dataset under `transformations` is in a separate file. +* Take a look at the sample under "sample_trips_{{ .project_name }}.py" to get familiar with the syntax. + Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. +* Use `Run file` to run and preview a single transformation. +* Use `Run pipeline` to run _all_ transformations in the entire pipeline. +* Use `+ Add` in the file browser to add a new data set definition. +* Use `Schedule` to run the pipeline on a schedule! + +For more tutorials and reference material, see https://docs.databricks.com/dlt. +{{ else -}} + +# {{template `pipeline_name` .}} + +This folder defines all source code for the '{{template `pipeline_name` .}}' pipeline: + +- `explorations`: Ad-hoc notebooks used to explore the data processed by this pipeline. +- `transformations`: All dataset definitions and transformations. +- `data_sources` (optional): View definitions describing the source data for this pipeline. + +## Getting Started + +To get started, go to the `transformations` folder -- most of the relevant source code lives there: + +* By convention, every dataset under `transformations` is in a separate file. +* Take a look at the sample under "sample_trips_{{ .project_name }}.sql" to get familiar with the syntax. + Read more about the syntax at https://docs.databricks.com/dlt/sql-ref.html. +* Use `Run file` to run and preview a single transformation. +* Use `Run pipeline` to run _all_ transformations in the entire pipeline. +* Use `+ Add` in the file browser to add a new data set definition. +* Use `Schedule` to run the pipeline on a schedule! + +For more tutorials and reference material, see https://docs.databricks.com/dlt. +{{ end -}} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/explorations/sample_exploration.ipynb.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/explorations/sample_exploration.ipynb.tmpl new file mode 100644 index 0000000000..967e663fae --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/explorations/sample_exploration.ipynb.tmpl @@ -0,0 +1,130 @@ +{{- if (eq .language "python") -}} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "19a992e9-55e0-49e4-abc7-8c92c420dd5b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "### Example Exploratory Notebook\n", + "\n", + "Use this notebook to explore the data generated by the pipeline in your preferred programming language.\n", + "\n", + "**Note**: This notebook is not executed as part of the pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1b0a82fa-3c6a-4f29-bb43-ded1c4fd77c6", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "# !!! Before performing any data analysis, make sure to run the pipeline to materialize the sample datasets. The tables referenced in this notebook depend on that step.\n", + "\n", + "display(spark.sql(\"SELECT * FROM {{ .default_catalog}}.{{template `static_dev_schema` .}}.{{ .project_name }}\"))" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "sample_exploration", + "widgets": {} + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} +{{ else -}} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3bd3cbb1-1518-4d0a-a8d1-f08da3f8840b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "### Example Exploratory Notebook\n", + "\n", + "Use this notebook to explore the data generated by the pipeline in your preferred programming language.\n", + "\n", + "**Note**: This notebook is not executed as part of the pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d30a8e05-bf7a-47e1-982e-b37e64cd6d43", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "-- !!! Before performing any data analysis, make sure to run the pipeline to materialize the sample datasets. The tables referenced in this notebook depend on that step.\n", + "\n", + "USE CATALOG `{{.default_catalog}}`;\n", + "USE SCHEMA `{{template `static_dev_schema` .}}`;\n", + "\n", + "SELECT * from {{ .project_name }};" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "sql", + "notebookMetadata": {}, + "notebookName": "sample_exploration", + "widgets": {} + }, + "language_info": { + "name": "sql" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} +{{ end -}} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py.tmpl new file mode 100644 index 0000000000..a191f88b9f --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py.tmpl @@ -0,0 +1,16 @@ +import dlt +from pyspark.sql.functions import col +from utilities import utils + + +# This file defines a sample transformation. +# Edit the sample below or add new transformations +# using "+ Add" in the file browser. + + +@dlt.table +def sample_trips_{{ .project_name }}(): + return ( + spark.read.table("samples.nyctaxi.trips") + .withColumn("trip_distance_km", utils.distance_km(col("trip_distance"))) + ) diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql.tmpl new file mode 100644 index 0000000000..b95a95da4d --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql.tmpl @@ -0,0 +1,9 @@ +-- This file defines a sample transformation. +-- Edit the sample below or add new transformations +-- using "+ Add" in the file browser. + +CREATE MATERIALIZED VIEW sample_trips_{{ .project_name }} AS +SELECT + pickup_zip, + fare_amount +FROM samples.nyctaxi.trips diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py.tmpl new file mode 100644 index 0000000000..64e40036d0 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py.tmpl @@ -0,0 +1,19 @@ +import dlt +from pyspark.sql.functions import col, sum + + +# This file defines a sample transformation. +# Edit the sample below or add new transformations +# using "+ Add" in the file browser. + + +@dlt.table +def sample_zones_{{ .project_name }}(): + # Read from the "sample_trips" table, then sum all the fares + return ( + spark.read.table("sample_trips_{{ .project_name }}") + .groupBy(col("pickup_zip")) + .agg( + sum("fare_amount").alias("total_fare") + ) + ) diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql.tmpl new file mode 100644 index 0000000000..ab84f4066a --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql.tmpl @@ -0,0 +1,10 @@ +-- This file defines a sample transformation. +-- Edit the sample below or add new transformations +-- using "+ Add" in the file browser. + +CREATE MATERIALIZED VIEW sample_zones_{{ .project_name }} AS +SELECT + pickup_zip, + SUM(fare_amount) AS total_fare +FROM sample_trips_{{ .project_name }} +GROUP BY pickup_zip diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py new file mode 100644 index 0000000000..ff039898f0 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py @@ -0,0 +1,8 @@ +from pyspark.sql.functions import udf +from pyspark.sql.types import FloatType + + +@udf(returnType=FloatType()) +def distance_km(distance_miles): + """Convert distance from miles to kilometers (1 mile = 1.60934 km).""" + return distance_miles * 1.60934 diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.job.yml.tmpl new file mode 100644 index 0000000000..1e7a7ca780 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.job.yml.tmpl @@ -0,0 +1,19 @@ +# The job that triggers {{template `pipeline_name` .}}. +resources: + jobs: + {{template `job_name` .}}: + name: {{template `job_name` .}} + + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS + + email_notifications: + on_failure: ${var.notifications} + + tasks: + - task_key: refresh_pipeline + pipeline_task: + pipeline_id: ${resources.pipelines.{{template `pipeline_name` .}}.id} diff --git a/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.pipeline.yml.tmpl new file mode 100644 index 0000000000..23df081f00 --- /dev/null +++ b/libs/template/templates/dlt/template/{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.pipeline.yml.tmpl @@ -0,0 +1,12 @@ +resources: + pipelines: + {{template `pipeline_name` .}}: + name: {{template `pipeline_name` .}} + serverless: true + channel: "PREVIEW" + catalog: ${var.catalog} + schema: ${var.schema} + root_path: "." + libraries: + - glob: + include: transformations/** diff --git a/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl b/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl index 5a2747d0f3..a3d8e27130 100644 --- a/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl +++ b/libs/template/templates/experimental-jobs-as-code/library/versions.tmpl @@ -6,4 +6,4 @@ >=15.4,<15.5 {{- end}} -{{define "latest_databricks_bundles_version" -}}0.254.0{{- end}} +{{define "latest_databricks_bundles_version" -}}0.255.0{{- end}} diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl index 70f56171c6..ec91f6d964 100644 --- a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/databricks.yml.tmpl @@ -26,6 +26,7 @@ artifacts: {{ end -}} include: - resources/*.yml + - resources/*/*.yml targets: dev: diff --git a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json index 53841d36f4..8fbc13c69f 100644 --- a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json +++ b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json @@ -14,7 +14,7 @@ "default": "{{default_catalog}}", "pattern": "^\\w*$", "pattern_match_failure_message": "Invalid catalog name.", - "description": "\nInitial catalog.\ndefault_catalog", + "description": "\nInitial catalog:\ndefault_catalog", "order": 3 }, "personal_schemas": { diff --git a/libs/testserver/dashboards.go b/libs/testserver/dashboards.go new file mode 100644 index 0000000000..5aab6e9909 --- /dev/null +++ b/libs/testserver/dashboards.go @@ -0,0 +1,96 @@ +package testserver + +import ( + "encoding/json" + "strings" + + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/google/uuid" +) + +func (s *FakeWorkspace) DashboardCreate(req Request) Response { + defer s.LockUnlock()() + + var dashboard dashboards.Dashboard + if err := json.Unmarshal(req.Body, &dashboard); err != nil { + return Response{ + StatusCode: 400, + } + } + + // Lakeview API strips hyphens from a uuid for dashboards + dashboard.DashboardId = strings.ReplaceAll(uuid.New().String(), "-", "") + + // All dashboards are active by default: + dashboard.LifecycleState = dashboards.LifecycleStateActive + + // Change path field if parent_path is provided + if dashboard.ParentPath != "" { + dashboard.Path = dashboard.ParentPath + "/" + dashboard.DisplayName + ".lvdash.json" + } + + // Parse serializedDashboard into json and put it back as a string + if dashboard.SerializedDashboard != "" { + var dashboardContent map[string]any + if err := json.Unmarshal([]byte(dashboard.SerializedDashboard), &dashboardContent); err == nil { + // Add pageType to each page in the pages array (as of June 2025, this is an undocumented Lakeview API behaviour) + if pages, ok := dashboardContent["pages"].([]any); ok { + for _, page := range pages { + if pageMap, ok := page.(map[string]any); ok { + pageMap["pageType"] = "PAGE_TYPE_CANVAS" + } + } + } + if updatedContent, err := json.Marshal(dashboardContent); err == nil { + dashboard.SerializedDashboard = string(updatedContent) + } + } + } + + s.Dashboards[dashboard.DashboardId] = dashboard + + return Response{ + Body: dashboards.Dashboard{ + DashboardId: dashboard.DashboardId, + Etag: uuid.New().String(), + }, + } +} + +func (s *FakeWorkspace) DashboardUpdate(req Request) Response { + defer s.LockUnlock()() + + var dashboard dashboards.Dashboard + if err := json.Unmarshal(req.Body, &dashboard); err != nil { + return Response{ + StatusCode: 400, + } + } + + // Update the etag for the dashboard. + dashboard.Etag = uuid.New().String() + + dashboardId := req.Vars["dashboard_id"] + s.Dashboards[dashboardId] = dashboard + + return Response{ + Body: dashboard, + } +} + +func (s *FakeWorkspace) DashboardPublish(req Request) Response { + defer s.LockUnlock()() + + var dashboard dashboards.Dashboard + if err := json.Unmarshal(req.Body, &dashboard); err != nil { + return Response{ + StatusCode: 400, + } + } + + return Response{ + Body: dashboards.PublishedDashboard{ + WarehouseId: dashboard.WarehouseId, + }, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 3b2baf9d55..68e872b794 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -40,10 +41,11 @@ type FakeWorkspace struct { Jobs map[int64]jobs.Job JobRuns map[int64]jobs.Run - Pipelines map[string]pipelines.GetPipelineResponse - Monitors map[string]catalog.MonitorInfo - Apps map[string]apps.App - Schemas map[string]catalog.SchemaInfo + Pipelines map[string]pipelines.GetPipelineResponse + Monitors map[string]catalog.MonitorInfo + Apps map[string]apps.App + Schemas map[string]catalog.SchemaInfo + Dashboards map[string]dashboards.Dashboard } func (w *FakeWorkspace) LockUnlock() func() { @@ -116,6 +118,7 @@ func NewFakeWorkspace(url string) *FakeWorkspace { Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, Schemas: map[string]catalog.SchemaInfo{}, + Dashboards: map[string]dashboards.Dashboard{}, } } diff --git a/main.go b/main.go index c568e6adbd..6af949d206 100644 --- a/main.go +++ b/main.go @@ -3,14 +3,28 @@ package main import ( "context" "os" + "path/filepath" + "runtime" "github.com/databricks/cli/cmd" + "github.com/databricks/cli/cmd/dlt" "github.com/databricks/cli/cmd/root" + "github.com/spf13/cobra" ) func main() { ctx := context.Background() - err := root.Execute(ctx, cmd.New(ctx)) + + // Branch command based on program name: 'dlt' runs DLT-specific commands, + // while 'databricks' runs the main CLI commands + invokedAs := filepath.Base(os.Args[0]) + var command *cobra.Command + if invokedAs == "dlt" || (runtime.GOOS == "windows" && invokedAs == "dlt.exe") { + command = dlt.New() + } else { + command = cmd.New(ctx) + } + err := root.Execute(ctx, command) if err != nil { os.Exit(1) } diff --git a/tools/lintdiff.py b/tools/lintdiff.py new file mode 100755 index 0000000000..cc859811fb --- /dev/null +++ b/tools/lintdiff.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Drop in replacement for golangci-lint that runs it only on changed packages. + +Changes are calculated as diff against main by default, use --ref or -H/--head to change this. +""" + +import os +import sys +import argparse +import subprocess + + +def parse_lines(cmd): + # print("+ " + " ".join(cmd), file=sys.stderr, flush=True) + result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8", check=True) + return [x.strip() for x in result.stdout.split("\n") if x.strip()] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--ref", default="main", help="Reference to calculate diff against.") + parser.add_argument("-H", "--head", action="store_true", help="Shortcut for '--ref HEAD' - test uncommitted changes only") + parser.add_argument("args", nargs=argparse.REMAINDER, help="golangci-lint command and options") + args = parser.parse_args() + + if not args.args: + args.args = ["run"] + + if args.head: + args.ref = "HEAD" + + gitroot = parse_lines(["git", "rev-parse", "--show-toplevel"])[0] + os.chdir(gitroot) + + changed = parse_lines(["git", "diff", "--name-only", args.ref, "--", "."]) + + # We need to pass packages to golangci-lint, not individual files. + dirs = set() + for filename in changed: + if "/testdata/" in filename: + continue + if filename.endswith(".go"): + d = os.path.dirname(filename) + dirs.add(d) + + if not dirs: + sys.exit(0) + + dirs = ["./" + d for d in sorted(dirs)] + + cmd = ["golangci-lint"] + args.args + dirs + print("+ " + " ".join(cmd), file=sys.stderr, flush=True) + os.execvp(cmd[0], cmd) + + +if __name__ == "__main__": + main() diff --git a/yamlfmt.yml b/yamlfmt.yml new file mode 100644 index 0000000000..992d58a1da --- /dev/null +++ b/yamlfmt.yml @@ -0,0 +1,11 @@ +exclude: + - libs/dyn/yamlloader/testdata + - acceptance/selftest/bundleconfig # https://github.com/google/yamlfmt/issues/254 + - acceptance/bundle/artifacts/nil_artifacts/databricks.yml # https://github.com/google/yamlfmt/issues/253 +formatter: + type: basic + retain_line_breaks_single: true + trim_trailing_whitespace: true + disable_alias_key_correction: true + scan_folded_as_literal: true + drop_merge_tag: true