apps MCP: restructure context management (#4073)

## Changes
Restructure the MCP context management into three layers to establish
separation of concerns.
Prompts are not changed here, subject for a separate change. 

## Why

###  Why separate context layers?

The previous implementation mixed different types of guidance in a
single template — app-specific validation rules sat alongside TypeScript
SDK patterns, making it hard to support new target types (jobs,
pipelines) or new templates (Python apps). By separating into L0 (tool
descriptions), L1 (universal workflow), L2 (target-specific), and L3
(template-specific), each layer can evolve independently. Adding a new
target type now means creating one target_<type>.tmpl file rather than
modifying shared templates.

### Why replace first-call middleware with explicit databricks_discover?

The middleware approach injected context into whatever tool happened to
run first — fragile and non-recoverable if the agent missed it. The
explicit databricks_discover tool lets agents request context when they
need it, re-call it when switching directories, and get project-aware
guidance (detecting apps/jobs from databricks.yml). The tool description
itself serves as L0 context, instructing agents to call it first during
planning without any middleware magic.

Author: arsenyinfo

experimental/apps-mcp/cmd/init_template.go

@@ -336,12 +336,12 @@ After initialization:
 			cmdio.LogString(ctx, fileTree)
 		}
 
-		// Try to read and display CLAUDE.md if present
-		readClaudeMd(ctx, configFile)
+		// Inject L2 (target-specific guidance for apps)
+		targetApps := prompts.MustExecuteTemplate("target_apps.tmpl", map[string]any{})
+		cmdio.LogString(ctx, targetApps)
 
-		// Re-inject app-specific guidance
-		appsContent := prompts.MustExecuteTemplate("apps.tmpl", map[string]any{})
-		cmdio.LogString(ctx, appsContent)
+		// Inject L3 (template-specific guidance from CLAUDE.md)
+		readClaudeMd(ctx, configFile)
 
 		return nil
 	}

experimental/apps-mcp/docs/context-management.md

@@ -0,0 +1,80 @@
+<!-- DO NOT MODIFY: This documentation defines the context management architecture for Databricks MCP -->
+# Context Management for Databricks MCP
+
+## Goals
+
+- Universal MCP for any coding agent (Claude, Cursor, etc.)
+- Support multiple target types: apps, jobs, pipelines
+- Support multiple templates per target type
+- Clean separation of context layers
+- Detect existing project context automatically
+
+## Context Layers
+
+| Layer | Content | When Injected |
+|-------|---------|---------------|
+| **L0: Tools** | Databricks MCP tool names and descriptions | Always (MCP protocol) |
+| **L1: Flow** | Universal workflow, available tools, CLI patterns | Always (via `databricks_discover`) |
+| **L2: Target** | Target-specific: validation, deployment, constraints | When target type detected or after `init-template` |
+| **L3: Template** | SDK/language-specific: file structure, commands, patterns | After `init-template`. For existing projects, agent reads CLAUDE.md. |
+
+L0 is implicit - tool descriptions guide agent behavior before any tool is called (e.g., `databricks_discover` description tells agent to call it first during planning).
+
+### Examples
+
+**L1 (universal):** "validate before deploying", "use invoke_databricks_cli for all commands"
+
+**L2 (apps):** app naming constraints, deployment consent requirement, app-specific validation
+
+**L3 (appkit-typescript):** npm scripts, tRPC patterns, useAnalyticsQuery usage, TypeScript import rules
+
+## Flows
+
+### New Project
+
+```
+Agent                           MCP
+  │                              │
+  ├─► databricks_discover        │
+  │   {working_directory: "."}   │
+  │                              ├─► Run detectors (nothing found)
+  │                              ├─► Return L1 only
+  │◄─────────────────────────────┤
+  │                              │
+  ├─► invoke_databricks_cli      │
+  │   ["...", "init-template", ...]
+  │                              ├─► Scaffold project
+  │                              ├─► Return L2[apps] + L3
+  │◄─────────────────────────────┤
+  │                              │
+  ├─► (agent now has L1 + L2 + L3)
+```
+
+### Existing Project
+
+```
+Agent                           MCP
+  │                              │
+  ├─► databricks_discover        │
+  │   {working_directory: "./my-app"}
+  │                              ├─► BundleDetector: found apps + jobs
+  │                              ├─► Return L1 + L2[apps] + L2[jobs]
+  │◄─────────────────────────────┤
+  │                              │
+  ├─► Read CLAUDE.md naturally   │
+  │   (agent learns L3 itself)   │
+```
+
+### Combined Bundles
+
+When `databricks.yml` contains multiple resource types (e.g., app + job), all relevant L2 layers are injected together.
+
+## Extensibility
+
+New target types can be added by:
+1. Creating `target_<type>.tmpl` in `lib/prompts/`
+2. Adding detection logic to recognize the target type from `databricks.yml`
+
+New templates can be added by:
+1. Creating template directory with CLAUDE.md (L3 guidance)
+2. Adding detection logic to recognize the template from project files

experimental/apps-mcp/lib/detector/bundle_detector.go

@@ -0,0 +1,54 @@
+package detector
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/phases"
+	"github.com/databricks/cli/libs/logdiag"
+)
+
+// BundleDetector detects Databricks bundle configuration.
+type BundleDetector struct{}
+
+// Detect loads databricks.yml with all includes and extracts target types.
+func (d *BundleDetector) Detect(ctx context.Context, workDir string, detected *DetectedContext) error {
+	bundlePath := filepath.Join(workDir, "databricks.yml")
+	if _, err := os.Stat(bundlePath); err != nil {
+		// no bundle file - not an error, just not a bundle project
+		return nil
+	}
+
+	// use full bundle loading to get all resources including from includes
+	ctx = logdiag.InitContext(ctx)
+	b, err := bundle.Load(ctx, workDir)
+	if err != nil || b == nil {
+		return nil
+	}
+
+	phases.Load(ctx, b)
+	if logdiag.HasError(ctx) {
+		return nil
+	}
+
+	detected.InProject = true
+	detected.BundleInfo = &BundleInfo{
+		Name:    b.Config.Bundle.Name,
+		RootDir: workDir,
+	}
+
+	// extract target types from fully loaded resources
+	if len(b.Config.Resources.Apps) > 0 {
+		detected.TargetTypes = append(detected.TargetTypes, "apps")
+	}
+	if len(b.Config.Resources.Jobs) > 0 {
+		detected.TargetTypes = append(detected.TargetTypes, "jobs")
+	}
+	if len(b.Config.Resources.Pipelines) > 0 {
+		detected.TargetTypes = append(detected.TargetTypes, "pipelines")
+	}
+
+	return nil
+}

experimental/apps-mcp/lib/detector/detector.go

@@ -0,0 +1,57 @@
+// Package detector provides project context detection for Databricks MCP.
+package detector
+
+import (
+	"context"
+)
+
+// BundleInfo contains information about a detected Databricks bundle.
+type BundleInfo struct {
+	Name    string
+	Target  string
+	RootDir string
+}
+
+// DetectedContext represents the detected project context.
+type DetectedContext struct {
+	InProject   bool
+	TargetTypes []string // ["apps", "jobs"] - supports combined bundles
+	Template    string   // "appkit-typescript", "python", etc.
+	BundleInfo  *BundleInfo
+	Metadata    map[string]string
+}
+
+// Detector detects project context from a working directory.
+type Detector interface {
+	// Detect examines the working directory and updates the context.
+	Detect(ctx context.Context, workDir string, detected *DetectedContext) error
+}
+
+// Registry manages a collection of detectors.
+type Registry struct {
+	detectors []Detector
+}
+
+// NewRegistry creates a new detector registry with default detectors.
+func NewRegistry() *Registry {
+	return &Registry{
+		detectors: []Detector{
+			&BundleDetector{},
+			&TemplateDetector{},
+		},
+	}
+}
+
+// Detect runs all detectors and returns the combined context.
+func (r *Registry) Detect(ctx context.Context, workDir string) *DetectedContext {
+	detected := &DetectedContext{
+		Metadata: make(map[string]string),
+	}
+
+	for _, d := range r.detectors {
+		// ignore errors - detectors should be resilient
+		_ = d.Detect(ctx, workDir, detected)
+	}
+
+	return detected
+}

experimental/apps-mcp/lib/detector/detector_test.go

@@ -0,0 +1,115 @@
+package detector_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/experimental/apps-mcp/lib/detector"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDetectorRegistry_EmptyDir(t *testing.T) {
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	registry := detector.NewRegistry()
+	detected := registry.Detect(ctx, dir)
+
+	assert.False(t, detected.InProject)
+	assert.Empty(t, detected.TargetTypes)
+	assert.Empty(t, detected.Template)
+}
+
+func TestDetectorRegistry_BundleWithApps(t *testing.T) {
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	bundleYml := `bundle:
+  name: my-app
+resources:
+  apps:
+    my_app: {}
+`
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644))
+
+	registry := detector.NewRegistry()
+	detected := registry.Detect(ctx, dir)
+
+	assert.True(t, detected.InProject)
+	assert.Equal(t, []string{"apps"}, detected.TargetTypes)
+	assert.Equal(t, "my-app", detected.BundleInfo.Name)
+}
+
+func TestDetectorRegistry_BundleWithJobs(t *testing.T) {
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	bundleYml := `bundle:
+  name: my-job
+resources:
+  jobs:
+    daily_job: {}
+`
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644))
+
+	registry := detector.NewRegistry()
+	detected := registry.Detect(ctx, dir)
+
+	assert.True(t, detected.InProject)
+	assert.Equal(t, []string{"jobs"}, detected.TargetTypes)
+	assert.Equal(t, "my-job", detected.BundleInfo.Name)
+}
+
+func TestDetectorRegistry_CombinedBundle(t *testing.T) {
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	bundleYml := `bundle:
+  name: my-project
+resources:
+  apps:
+    my_app: {}
+  jobs:
+    daily_job: {}
+`
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644))
+
+	registry := detector.NewRegistry()
+	detected := registry.Detect(ctx, dir)
+
+	assert.True(t, detected.InProject)
+	assert.Contains(t, detected.TargetTypes, "apps")
+	assert.Contains(t, detected.TargetTypes, "jobs")
+	assert.Equal(t, "my-project", detected.BundleInfo.Name)
+}
+
+func TestDetectorRegistry_AppkitTemplate(t *testing.T) {
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	// bundle + package.json with appkit marker
+	bundleYml := `bundle:
+  name: my-app
+resources:
+  apps:
+    my_app: {}
+`
+	packageJson := `{
+  "name": "my-app",
+  "dependencies": {
+    "@databricks/sql": "^1.0.0"
+  }
+}`
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYml), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "package.json"), []byte(packageJson), 0o644))
+
+	registry := detector.NewRegistry()
+	detected := registry.Detect(ctx, dir)
+
+	assert.True(t, detected.InProject)
+	assert.Equal(t, []string{"apps"}, detected.TargetTypes)
+	assert.Equal(t, "appkit-typescript", detected.Template)
+}