[1/5] Aitools: state tracking, manifest source, directory rename (#4810)

## PR Stack

This is part of the aitools feature-complete initiative. PRs should be
reviewed and merged in order.

1. **[1/5] State + release discovery + directory rename** (this PR)
2. [2/5] Install writes state + interactive agent selection (#4811)
3. [3/5] Update + uninstall + version commands (#4812)
4. [4/5] List improvements + command restructuring + flags (#4813)
5. [5/5] Project scope (--project/--global) (pending)

Manifest v2 PR: (pending in databricks-agent-skills)

## Why

The aitools skills installer has no state tracking. There's no record of
what was installed, at what version, or when. This blocks every
lifecycle command (update, uninstall, version). The manifest fetching
logic is also hardcoded with no abstraction for testing.

## Changes

Before: Skills install to `~/.databricks/agent-skills/` with no state
file. `FetchManifest` is inlined in `installer.go` with a hardcoded ref
and no way to mock it in tests.

Now:
- `InstallState` type with `LoadState`/`SaveState` (atomic write via
temp+rename) for tracking installed skills, release ref, and timestamps
- `ManifestSource` interface with `GitHubManifestSource`, separating
manifest fetching from install logic. Enables test mocking.
- `FetchLatestRelease` to discover newest release from GitHub API. Falls
back to pinned ref on network failure.
- v2 fields on `SkillMeta` (`Experimental`, `Description`, `MinCLIVer`)
with omitempty for backward compat with v1 manifests
- Canonical directory renamed from `.databricks/agent-skills` to
`.databricks/aitools/skills`
- Backward-compat check in `HasDatabricksSkillsInstalled` for the old
path

No behavior changes to existing commands except the directory path.

## Test plan

- [x] `LoadState` returns nil, nil for nonexistent file
- [x] `SaveState` + `LoadState` roundtrip preserves all fields including
optional ones
- [x] `SaveState` creates nested directories, writes trailing newline
- [x] `LoadState` returns error for corrupt JSON
- [x] `GlobalSkillsDir` resolves correctly
- [x] `ProjectSkillsDir` returns ErrNotImplemented
- [x] `HasDatabricksSkillsInstalled` detects legacy path
- [x] All existing aitools tests still pass

---------

Co-authored-by: Arseny Kravchenko <arsenyinfo@users.noreply.github.com>

Author: simonfaltum

experimental/aitools/lib/agents/skills.go

@@ -14,12 +14,15 @@ const (
 	databricksSkillPrefix = "databricks"
 
 	// CanonicalSkillsDir is the shared location for skills when multiple agents are detected.
-	CanonicalSkillsDir = ".databricks/agent-skills"
+	CanonicalSkillsDir = ".databricks/aitools/skills"
+
+	// legacySkillsDir is the old canonical location, checked for backward compatibility.
+	legacySkillsDir = ".databricks/agent-skills"
 )
 
 // HasDatabricksSkillsInstalled checks if Databricks skills are installed in the canonical location.
-// Returns true if no agents are detected (nothing to recommend) or if skills exist in ~/.databricks/agent-skills/.
-// Only the canonical location is checked so that skills installed by other tools are not mistaken for a proper installation.
+// Returns true if no agents are detected (nothing to recommend) or if skills exist in
+// ~/.databricks/aitools/skills/ or the legacy ~/.databricks/agent-skills/.
 func HasDatabricksSkillsInstalled(ctx context.Context) bool {
 	installed := DetectInstalled(ctx)
 	if len(installed) == 0 {
@@ -30,7 +33,8 @@ func HasDatabricksSkillsInstalled(ctx context.Context) bool {
 	if err != nil {
 		return false
 	}
-	return hasDatabricksSkillsIn(filepath.Join(homeDir, CanonicalSkillsDir))
+	return hasDatabricksSkillsIn(filepath.Join(homeDir, CanonicalSkillsDir)) ||
+		hasDatabricksSkillsIn(filepath.Join(homeDir, legacySkillsDir))
 }
 
 // hasDatabricksSkillsIn checks if dir contains a subdirectory starting with "databricks".

experimental/aitools/lib/agents/skills_test.go

@@ -134,3 +134,25 @@ func TestHasDatabricksSkillsInstalledDatabricksAppsCanonical(t *testing.T) {
 
 	assert.True(t, HasDatabricksSkillsInstalled(t.Context()))
 }
+
+func TestHasDatabricksSkillsInstalledLegacyPath(t *testing.T) {
+	tmpHome := t.TempDir()
+	t.Setenv("HOME", tmpHome)
+	// Skills only in the legacy location should still be detected.
+	require.NoError(t, os.MkdirAll(filepath.Join(tmpHome, legacySkillsDir, "databricks"), 0o755))
+
+	agentDir := filepath.Join(tmpHome, ".claude")
+	require.NoError(t, os.MkdirAll(agentDir, 0o755))
+
+	origRegistry := Registry
+	Registry = []Agent{
+		{
+			Name:        "test-agent",
+			DisplayName: "Test Agent",
+			ConfigDir:   func(_ context.Context) (string, error) { return agentDir, nil },
+		},
+	}
+	defer func() { Registry = origRegistry }()
+
+	assert.True(t, HasDatabricksSkillsInstalled(t.Context()))
+}

experimental/aitools/lib/installer/installer.go

@@ -2,7 +2,6 @@ package installer
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,7 +12,6 @@ import (
 	"github.com/databricks/cli/experimental/aitools/lib/agents"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/env"
-	"github.com/databricks/cli/libs/log"
 	"github.com/fatih/color"
 )
 
@@ -40,39 +38,20 @@ type Manifest struct {
 
 // SkillMeta describes a single skill entry in the manifest.
 type SkillMeta struct {
-	Version   string   `json:"version"`
-	UpdatedAt string   `json:"updated_at"`
-	Files     []string `json:"files"`
+	Version      string   `json:"version"`
+	UpdatedAt    string   `json:"updated_at"`
+	Files        []string `json:"files"`
+	Experimental bool     `json:"experimental,omitempty"`
+	Description  string   `json:"description,omitempty"`
+	MinCLIVer    string   `json:"min_cli_version,omitempty"`
 }
 
 // FetchManifest fetches the skills manifest from the skills repo.
+// This is a convenience wrapper that uses the default GitHubManifestSource.
 func FetchManifest(ctx context.Context) (*Manifest, error) {
+	src := &GitHubManifestSource{}
 	ref := getSkillsRef(ctx)
-	log.Infof(ctx, "Fetching skills manifest from %s/%s@%s", skillsRepoOwner, skillsRepoName, ref)
-	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/manifest.json",
-		skillsRepoOwner, skillsRepoName, ref)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request: %w", err)
-	}
-
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch manifest: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode)
-	}
-
-	var manifest Manifest
-	if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
-		return nil, fmt.Errorf("failed to parse manifest: %w", err)
-	}
-
-	return &manifest, nil
+	return src.FetchManifest(ctx, ref)
 }
 
 func fetchSkillFile(ctx context.Context, skillName, filePath string) ([]byte, error) {

experimental/aitools/lib/installer/source.go

@@ -0,0 +1,50 @@
+package installer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/databricks/cli/libs/log"
+)
+
+// ManifestSource abstracts how the skills manifest is fetched.
+type ManifestSource interface {
+	// FetchManifest fetches the skills manifest at the given ref.
+	FetchManifest(ctx context.Context, ref string) (*Manifest, error)
+}
+
+// GitHubManifestSource fetches manifests from GitHub.
+type GitHubManifestSource struct{}
+
+// FetchManifest fetches the skills manifest from GitHub at the given ref.
+func (s *GitHubManifestSource) FetchManifest(ctx context.Context, ref string) (*Manifest, error) {
+	log.Infof(ctx, "Fetching skills manifest from %s/%s@%s", skillsRepoOwner, skillsRepoName, ref)
+	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/manifest.json",
+		skillsRepoOwner, skillsRepoName, ref)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch manifest: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode)
+	}
+
+	var manifest Manifest
+	if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
+		return nil, fmt.Errorf("failed to parse manifest: %w", err)
+	}
+
+	return &manifest, nil
+}

experimental/aitools/lib/installer/state.go

@@ -0,0 +1,98 @@
+package installer
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/databricks/cli/libs/env"
+)
+
+const stateFileName = ".state.json"
+
+// ErrNotImplemented indicates that a feature is not yet implemented.
+var ErrNotImplemented = errors.New("project scope not yet implemented")
+
+// InstallState records the state of all installed skills in a scope directory.
+type InstallState struct {
+	SchemaVersion       int               `json:"schema_version"`
+	IncludeExperimental bool              `json:"include_experimental,omitempty"`
+	Release             string            `json:"release"`
+	LastUpdated         time.Time         `json:"last_updated"`
+	Skills              map[string]string `json:"skills"`
+	Scope               string            `json:"scope,omitempty"`
+}
+
+// LoadState reads install state from the given directory.
+// Returns (nil, nil) when the state file does not exist.
+func LoadState(dir string) (*InstallState, error) {
+	data, err := os.ReadFile(filepath.Join(dir, stateFileName))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read state file: %w", err)
+	}
+
+	var state InstallState
+	if err := json.Unmarshal(data, &state); err != nil {
+		return nil, fmt.Errorf("failed to parse state file: %w", err)
+	}
+	return &state, nil
+}
+
+// SaveState writes install state to the given directory atomically.
+// Creates the directory if it does not exist.
+func SaveState(dir string, state *InstallState) error {
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("failed to create state directory: %w", err)
+	}
+
+	data, err := json.MarshalIndent(state, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal state: %w", err)
+	}
+	data = append(data, '\n')
+
+	// Atomic write: write to temp file in the same directory, then rename.
+	tmp, err := os.CreateTemp(dir, ".state-*.tmp")
+	if err != nil {
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	tmpName := tmp.Name()
+
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		os.Remove(tmpName)
+		return fmt.Errorf("failed to write temp file: %w", err)
+	}
+	if err := tmp.Close(); err != nil {
+		os.Remove(tmpName)
+		return fmt.Errorf("failed to close temp file: %w", err)
+	}
+
+	if err := os.Rename(tmpName, filepath.Join(dir, stateFileName)); err != nil {
+		os.Remove(tmpName)
+		return fmt.Errorf("failed to rename state file: %w", err)
+	}
+	return nil
+}
+
+// GlobalSkillsDir returns the path to the global skills directory (~/.databricks/aitools/skills/).
+func GlobalSkillsDir(ctx context.Context) (string, error) {
+	home, err := env.UserHomeDir(ctx)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, ".databricks", "aitools", "skills"), nil
+}
+
+// ProjectSkillsDir returns the path to the project-scoped skills directory.
+// Project scope is not yet implemented.
+func ProjectSkillsDir(_ context.Context) (string, error) {
+	return "", ErrNotImplemented
+}

experimental/aitools/lib/installer/state_test.go

@@ -0,0 +1,116 @@
+package installer
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/libs/env"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadStateNonexistentFile(t *testing.T) {
+	state, err := LoadState(t.TempDir())
+	assert.NoError(t, err)
+	assert.Nil(t, state)
+}
+
+func TestSaveAndLoadStateRoundtrip(t *testing.T) {
+	dir := t.TempDir()
+	original := &InstallState{
+		SchemaVersion: 1,
+		Release:       "v0.2.0",
+		LastUpdated:   time.Date(2026, 3, 22, 10, 0, 0, 0, time.UTC),
+		Skills: map[string]string{
+			"databricks": "1.0.0",
+		},
+	}
+
+	err := SaveState(dir, original)
+	require.NoError(t, err)
+
+	loaded, err := LoadState(dir)
+	require.NoError(t, err)
+	assert.Equal(t, original, loaded)
+}
+
+func TestSaveStateCreatesDirectory(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "nested", "path")
+	state := &InstallState{
+		SchemaVersion: 1,
+		Release:       "v0.1.0",
+		LastUpdated:   time.Date(2026, 3, 22, 9, 0, 0, 0, time.UTC),
+		Skills:        map[string]string{},
+	}
+
+	err := SaveState(dir, state)
+	require.NoError(t, err)
+
+	// Verify file exists.
+	_, err = os.Stat(filepath.Join(dir, stateFileName))
+	assert.NoError(t, err)
+}
+
+func TestSaveStateTrailingNewline(t *testing.T) {
+	dir := t.TempDir()
+	state := &InstallState{
+		SchemaVersion: 1,
+		Release:       "v0.1.0",
+		LastUpdated:   time.Date(2026, 3, 22, 9, 0, 0, 0, time.UTC),
+		Skills:        map[string]string{},
+	}
+
+	err := SaveState(dir, state)
+	require.NoError(t, err)
+
+	data, err := os.ReadFile(filepath.Join(dir, stateFileName))
+	require.NoError(t, err)
+	assert.Equal(t, byte('\n'), data[len(data)-1])
+}
+
+func TestLoadStateCorruptJSON(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, stateFileName), []byte("{bad json"), 0o644))
+
+	state, err := LoadState(dir)
+	assert.Error(t, err)
+	assert.Nil(t, state)
+	assert.Contains(t, err.Error(), "failed to parse state file")
+}
+
+func TestGlobalSkillsDir(t *testing.T) {
+	ctx := env.WithUserHomeDir(t.Context(), "/fake/home")
+	dir, err := GlobalSkillsDir(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join("/fake/home", ".databricks", "aitools", "skills"), dir)
+}
+
+func TestProjectSkillsDirNotImplemented(t *testing.T) {
+	dir, err := ProjectSkillsDir(t.Context())
+	assert.ErrorIs(t, err, ErrNotImplemented)
+	assert.Empty(t, dir)
+}
+
+func TestSaveAndLoadStateWithOptionalFields(t *testing.T) {
+	dir := t.TempDir()
+	original := &InstallState{
+		SchemaVersion:       1,
+		IncludeExperimental: true,
+		Release:             "v0.3.0",
+		LastUpdated:         time.Date(2026, 3, 22, 12, 30, 0, 0, time.UTC),
+		Skills: map[string]string{
+			"databricks": "1.0.0",
+			"sql-tools":  "0.2.0",
+		},
+		Scope: "project",
+	}
+
+	err := SaveState(dir, original)
+	require.NoError(t, err)
+
+	loaded, err := LoadState(dir)
+	require.NoError(t, err)
+	assert.Equal(t, original, loaded)
+}