DLT init reusing lakeflow template

Author: alyssa-db

cmd/dlt/dlt.go

@@ -1,9 +1,12 @@
 package dlt
 
 import (
-	"context"
+	"log/slog"
 
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/log/handler"
 	"github.com/spf13/cobra"
 )
 
@@ -12,21 +15,28 @@ func New() *cobra.Command {
 		Use:   "dlt",
 		Short: "DLT CLI",
 		Long:  "DLT CLI (stub, to be filled in)",
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			// Initialize cmdio context
+			cmdIO := cmdio.NewIO(cmd.Context(), flags.OutputText, cmd.InOrStdin(), cmd.OutOrStdout(), cmd.ErrOrStderr(), "", "")
+			ctx := cmdio.InContext(cmd.Context(), cmdIO)
+
+			// Set up logger with WARN level
+			h := handler.NewFriendlyHandler(cmd.ErrOrStderr(), &handler.Options{
+				Color: cmdio.IsTTY(cmd.ErrOrStderr()),
+				Level: log.LevelWarn,
+			})
+			logger := slog.New(h)
+			ctx = log.NewContext(ctx, logger)
+
+			cmd.SetContext(ctx)
+			return nil
+		},
 		Run: func(cmd *cobra.Command, args []string) {
 			_ = cmd.Help()
 		},
 	}
 
-	// Add 'init' stub command (same description as bundle init)
-	initCmd := &cobra.Command{
-		Use:   "init",
-		Short: "Initialize a new DLT project in the current directory",
-		Long:  "Initialize a new DLT project in the current directory. This is a stub for future implementation.",
-		Run: func(cmd *cobra.Command, args []string) {
-			cmdio.LogString(context.Background(), "dlt init is not yet implemented. This will initialize a new DLT project in the future.")
-		},
-	}
-	cmd.AddCommand(initCmd)
+	cmd.AddCommand(initCommand())
 
 	return cmd
 }

libs/template/templates/lakeflow-pipelines/databricks_template_schema.json

@@ -9,6 +9,12 @@
             "pattern": "^[a-z0-9_]+$",
             "pattern_match_failure_message": "Name must consist of lower case letters, numbers, and underscores."
         },
+        "is_dlt": {
+            "skip_prompt_if": {},
+            "type": "boolean",
+            "description": "DLT pipelines using this template",
+            "default": false
+        },
         "default_catalog": {
             "type": "string",
             "default": "{{default_catalog}}",

libs/template/templates/lakeflow-pipelines/template/__preamble.tmpl

@@ -5,6 +5,7 @@ This file only contains template directives; it is skipped for the actual output
 {{skip "__preamble"}}
 
 {{$isSQL := eq .language "sql"}}
+{{$isDLT := eq .is_dlt true}}
 
 {{if $isSQL}}
   {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py"}}
@@ -14,3 +15,7 @@ This file only contains template directives; it is skipped for the actual output
   {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql"}}
   {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql"}}
 {{end}}
+
+{{if $isDLT}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/{{.project_name}}.job.yml"}}
+{{end}}

libs/template/templates/lakeflow-pipelines/template/{{.project_name}}/databricks.yml.tmpl

@@ -1,12 +1,12 @@
 # This is a Databricks asset bundle definition for {{.project_name}}.
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
-bundle:
+{{$isDLT := eq .is_dlt true}}{{if $isDLT}}project{{else}}bundle{{end}}:
   name: {{.project_name}}
-  uuid: {{bundle_uuid}}
+  {{if not $isDLT}}uuid: {{bundle_uuid}}{{end}}
 
 include:
-  - resources/*.yml
-  - resources/*/*.yml
+  {{if $isDLT}}- ./*.yml{{else}}- resources/*.yml
+  - resources/*/*.yml{{end}}
 
 # Variable declarations. These variables are assigned in the dev/prod targets below.
 variables:
@@ -25,6 +25,7 @@ targets:
     # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
     mode: development
     default: true
+    {{if $isDLT}}deploy_on_run: true{{end}}
     workspace:
       host: {{workspace_host}}
     variables:
@@ -36,11 +37,11 @@ targets:
     mode: production
     workspace:
       host: {{workspace_host}}
-      # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy.
-      root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
-    permissions:
+      {{if not $isDLT}}# We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy.
+      root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}}{{end}}
+    {{if $isDLT}}owner: user@company.com{{else}}permissions:
       - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
-        level: CAN_MANAGE
+        level: CAN_MANAGE{{end}}
     variables:
       catalog: {{.default_catalog}}
       schema: {{template `prod_schema` .}}

libs/template/writer.go

@@ -32,11 +32,20 @@ type Writer interface {
 	// 2. The output directory where the template will be materialized.
 	Configure(ctx context.Context, configPath, outputDir string) error
 
+	// Finalize walks the template file tree, persists files to disk, and prints the success message.
+	Finalize(ctx context.Context) error
+
 	// Materialize the template to the local file system.
 	Materialize(ctx context.Context, r Reader) error
 
 	// Log telemetry for the template initialization event.
 	LogTelemetry(ctx context.Context)
+
+	// SetConfig sets a value in the template's config.
+	SetConfig(key string, value any)
+
+	// PromptForInput prompts the user for any missing config values.
+	PromptForInput(ctx context.Context, reader Reader) error
 }
 
 type defaultWriter struct {
@@ -49,6 +58,13 @@ type defaultWriter struct {
 	renderer *renderer
 }
 
+// SetConfig sets a value in the template's config.
+func (tmpl *defaultWriter) SetConfig(key string, value any) {
+	if tmpl.renderer != nil {
+		tmpl.renderer.config[key] = value
+	}
+}
+
 func constructOutputFiler(ctx context.Context, outputDir string) (filer.Filer, error) {
 	outputDir, err := filepath.Abs(outputDir)
 	if err != nil {
@@ -81,7 +97,7 @@ func (tmpl *defaultWriter) Configure(ctx context.Context, configPath, outputDir
 	return nil
 }
 
-func (tmpl *defaultWriter) promptForInput(ctx context.Context, reader Reader) error {
+func (tmpl *defaultWriter) PromptForInput(ctx context.Context, reader Reader) error {
 	readerFs, err := reader.FS(ctx)
 	if err != nil {
 		return err
@@ -143,15 +159,10 @@ func (tmpl *defaultWriter) printSuccessMessage(ctx context.Context) error {
 	return nil
 }
 
-func (tmpl *defaultWriter) Materialize(ctx context.Context, reader Reader) error {
-	err := tmpl.promptForInput(ctx, reader)
-	if err != nil {
-		return err
-	}
+func (tmpl *defaultWriter) Finalize(ctx context.Context) error {
+	// Walk the template file tree and compute in-memory representations of the output files.
 
-	// Walk the template file tree and compute in-memory representations of the
-	// output files.
-	err = tmpl.renderer.walk()
+	err := tmpl.renderer.walk()
 	if err != nil {
 		return err
 	}
@@ -165,6 +176,15 @@ func (tmpl *defaultWriter) Materialize(ctx context.Context, reader Reader) error
 	return tmpl.printSuccessMessage(ctx)
 }
 
+func (tmpl *defaultWriter) Materialize(ctx context.Context, reader Reader) error {
+	err := tmpl.PromptForInput(ctx, reader)
+	if err != nil {
+		return err
+	}
+
+	return tmpl.Finalize(ctx)
+}
+
 func (tmpl *defaultWriter) LogTelemetry(ctx context.Context) {
 	telemetry.Log(ctx, protos.DatabricksCliLog{
 		BundleInitEvent: &protos.BundleInitEvent{
@@ -178,6 +198,10 @@ type writerWithFullTelemetry struct {
 	defaultWriter
 }
 
+func (tmpl *writerWithFullTelemetry) SetConfig(key string, value any) {
+	tmpl.defaultWriter.SetConfig(key, value)
+}
+
 func (tmpl *writerWithFullTelemetry) LogTelemetry(ctx context.Context) {
 	var args []protos.BundleInitTemplateEnumArg
 	for k, v := range tmpl.config.values {
@@ -216,3 +240,7 @@ func (tmpl *writerWithFullTelemetry) LogTelemetry(ctx context.Context) {
 		},
 	})
 }
+
+func (tmpl *writerWithFullTelemetry) Finalize(ctx context.Context) error {
+	return tmpl.defaultWriter.Finalize(ctx)
+}