diff --git a/Dockerfile b/Dockerfile index b7ff9bae..89e918e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ FROM alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2 ARG GOSEC_VERSION ARG SEMGREP_VERSION -RUN apk add --no-cache git ca-certificates curl wget python3 python3-dev py3-pip alpine-sdk clamav +RUN apk add --no-cache git ca-certificates curl wget python3 python3-dev py3-pip alpine-sdk clamav nodejs npm RUN update-ca-certificates RUN freshclam diff --git a/pkg/analysis/passes/codediff/codediff.go b/pkg/analysis/passes/codediff/codediff.go index 37002f34..035b6e8f 100644 --- a/pkg/analysis/passes/codediff/codediff.go +++ b/pkg/analysis/passes/codediff/codediff.go @@ -137,8 +137,7 @@ func run(pass *analysis.Pass) (any, error) { return nil, nil } - geminiKey := os.Getenv("GEMINI_API_KEY") - if geminiKey == "" { + if err := llmClient.CanUseLLM(); err != nil { return nil, nil } @@ -303,19 +302,10 @@ func runLLMAnalysis( return nil, err } - // clean up files from repositoryPath - cleanFiles := []string{"replies.json", ".nvmrc", "GEMINI.md"} - for _, file := range cleanFiles { - filePath := filepath.Join(repositoryPath, file) - if _, err := os.Stat(filePath); err == nil { - if err := os.Remove(filePath); err != nil { - logme.Debugln("Failed to remove file:", err) - } - } - } + llmclient.CleanUpPromptFiles(repositoryPath) // Call the LLM - if err := llmClient.CallLLM(prompt, repositoryPath); err != nil { + if err := llmClient.CallLLM(prompt, repositoryPath, nil); err != nil { logme.Debugln("Failed to call LLM:", err) return nil, err } diff --git a/pkg/llmclient/client.go b/pkg/llmclient/client.go index 6e7390a3..a70f325c 100644 --- a/pkg/llmclient/client.go +++ b/pkg/llmclient/client.go @@ -2,17 +2,65 @@ package llmclient import ( "context" + _ "embed" "errors" + "fmt" "os" "os/exec" + "path/filepath" "strings" "time" "github.com/grafana/plugin-validator/pkg/logme" ) +//go:embed settings.json +var embeddedSettings []byte + +var ErrAPIKeyNotSet = errors.New("GEMINI_API_KEY not set") + +// filesToClean are files that should be removed from the working directory +// before calling the LLM to avoid influencing its behavior. +var filesToClean = []string{ + "GEMINI.md", "gemini.md", + "CLAUDE.md", "claude.md", + "AGENTS.md", "agents.md", + "COPILOT.md", "copilot.md", + "replies.json", + "output.json", +} + +// CleanUpPromptFiles removes agent config files and known output files +// from the given directory to avoid influencing the LLM. +func CleanUpPromptFiles(dir string) { + for _, file := range filesToClean { + p := filepath.Join(dir, file) + if _, err := os.Stat(p); err == nil { + if err := os.Remove(p); err != nil { + logme.DebugFln("Failed to remove %s: %v", p, err) + } + } + } +} + +type CallLLMOptions struct { + Model string // e.g. "gemini-2.5-flash", empty = CLI default +} + type LLMClient interface { - CallLLM(prompt, repositoryPath string) error + CanUseLLM() error + CallLLM(prompt, repositoryPath string, opts *CallLLMOptions) error +} + +func (g *GeminiClient) CanUseLLM() error { + if os.Getenv("GEMINI_API_KEY") == "" { + return ErrAPIKeyNotSet + } + _, err := getGeminiBinaryPath() + if err != nil { + return err + } + return nil } type GeminiClient struct{} @@ -21,40 +69,113 @@ func NewGeminiClient() *GeminiClient { return &GeminiClient{} } -func (g *GeminiClient) CallLLM(prompt, repositoryPath string) error { - _, err := exec.LookPath("npx") +var cachedGeminiBinPath string +var cachedSettingsPath string + +// getSettingsPath writes the embedded settings.json to a temp file once and returns its path. +func getSettingsPath() (string, error) { + if cachedSettingsPath != "" { + return cachedSettingsPath, nil + } + + dir, err := os.MkdirTemp("", "gemini-settings-*") if err != nil { - return errors.New("npx is not available in PATH") + return "", fmt.Errorf("failed to create temp settings dir: %w", err) + } + + p := filepath.Join(dir, "settings.json") + if err := os.WriteFile(p, embeddedSettings, 0644); err != nil { + os.RemoveAll(dir) + return "", fmt.Errorf("failed to write settings file: %w", err) + } + + cachedSettingsPath = p + logme.DebugFln("Gemini settings written to %s", cachedSettingsPath) + return cachedSettingsPath, nil +} + +// getGeminiBinaryPath returns the path to the gemini binary. +// It first checks PATH, then falls back to a local npm install. +// The result is cached after the first successful resolution. +func getGeminiBinaryPath() (string, error) { + if cachedGeminiBinPath != "" { + return cachedGeminiBinPath, nil + } + + if p, err := exec.LookPath("gemini"); err == nil { + cachedGeminiBinPath = p + return p, nil + } + + if _, err := exec.LookPath("npm"); err != nil { + return "", fmt.Errorf("neither gemini nor npm available in PATH") + } + + dir, err := os.MkdirTemp("", "gemini-cli-*") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %w", err) + } + + logme.DebugFln("Installing gemini CLI locally to %s", dir) + install := exec.Command("npm", "install", "@google/gemini-cli") + install.Dir = dir + if out, err := install.CombinedOutput(); err != nil { + os.RemoveAll(dir) + return "", fmt.Errorf("npm install failed: %s: %w", string(out), err) + } + + bin := filepath.Join(dir, "node_modules", ".bin", "gemini") + if _, err := os.Stat(bin); err != nil { + os.RemoveAll(dir) + return "", fmt.Errorf("gemini binary not found after install") + } + + cachedGeminiBinPath = bin + logme.DebugFln("Gemini CLI installed at %s", bin) + return bin, nil +} + +func (g *GeminiClient) CallLLM(prompt, repositoryPath string, opts *CallLLMOptions) error { + if err := g.CanUseLLM(); err != nil { + return err + } + + geminiBin, err := getGeminiBinaryPath() + if err != nil { + return fmt.Errorf("failed to get gemini CLI: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - cmd := exec.CommandContext( - ctx, - "npx", - "-y", - "https://github.com/google-gemini/gemini-cli", - "-y", - ) + settingsPath, err := getSettingsPath() + if err != nil { + return fmt.Errorf("failed to prepare settings: %w", err) + } + + args := []string{} + + if opts != nil && opts.Model != "" { + args = append(args, "-m", opts.Model) + } + + cmd := exec.CommandContext(ctx, geminiBin, args...) cmd.Dir = repositoryPath cmd.Stdin = strings.NewReader(prompt) - // we only want the output in debug mode + cmd.Env = append(os.Environ(), "GEMINI_CLI_SYSTEM_SETTINGS_PATH="+settingsPath) + if os.Getenv("DEBUG") != "" { cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr } - logme.Debugln("Running gemini CLI analysis in directory:", repositoryPath) + logme.DebugFln("Running: GEMINI_CLI_SYSTEM_SETTINGS_PATH=%s %s %s", settingsPath, geminiBin, strings.Join(args, " ")) if err := cmd.Run(); err != nil { if ctx.Err() == context.DeadlineExceeded { - logme.Debugln("Gemini CLI timed out after 5 minutes") - } else { - logme.Debugln("Gemini CLI failed:", err) + return fmt.Errorf("gemini CLI timed out after 5 minutes") } + return fmt.Errorf("gemini CLI failed: %w", err) } return nil } - diff --git a/pkg/llmclient/mock.go b/pkg/llmclient/mock.go index b0c643ed..603253e1 100644 --- a/pkg/llmclient/mock.go +++ b/pkg/llmclient/mock.go @@ -46,7 +46,11 @@ func (m *MockLLMClient) WithResponses(responses []MockResponse) *MockLLMClient { return m } -func (m *MockLLMClient) CallLLM(prompt, repositoryPath string) error { +func (m *MockLLMClient) CanUseLLM() error { + return nil +} + +func (m *MockLLMClient) CallLLM(prompt, repositoryPath string, opts *CallLLMOptions) error { logme.Debugln("Mock LLM client called with repository:", repositoryPath) repliesPath := filepath.Join(repositoryPath, "replies.json") diff --git a/pkg/llmclient/settings.json b/pkg/llmclient/settings.json new file mode 100644 index 00000000..d7644859 --- /dev/null +++ b/pkg/llmclient/settings.json @@ -0,0 +1,45 @@ +{ + "skills": { + "enabled": false + }, + "tools": { + "approvalMode": "auto_edit", + "allowed": [ + "ShellTool(ls)", + "ShellTool(cat)", + "ShellTool(git status)", + "ShellTool(git diff)", + "ShellTool(git log)", + "ShellTool(git show)", + "ShellTool(git blame)", + "ShellTool(git branch)", + "ShellTool(git tag)", + "ShellTool(git remote)", + "ShellTool(git rev-parse)", + "ShellTool(git ls-files)", + "ShellTool(git shortlog)", + "ShellTool(git checkout)", + "ShellTool(git reset)", + "ShellTool(git clean)", + "ShellTool(grep)", + "ShellTool(rg)", + "ShellTool(head)", + "ShellTool(tail)", + "ShellTool(wc)", + "ShellTool(diff)", + "ShellTool(tree)", + "ShellTool(file)", + "ShellTool(stat)", + "ShellTool(sort)", + "ShellTool(uniq)", + "ShellTool(jq)" + ], + "exclude": [ + "web_fetch", + "google_web_search", + "ShellTool(curl)", + "ShellTool(wget)", + "ShellTool(ssh)" + ] + } +} diff --git a/pkg/repotool/github.go b/pkg/repotool/github.go index a919cf8f..088373d8 100644 --- a/pkg/repotool/github.go +++ b/pkg/repotool/github.go @@ -211,7 +211,7 @@ func FindReleaseByVersion( CommitSHA: commitSHA, Source: "github_release", CreatedAt: createdAt, - URL: release.HTMLURL, + URL: fmt.Sprintf("https://github.com/%s/%s/commit/%s", repo.Owner, repo.Repo, commitSHA), }, nil } } @@ -226,18 +226,12 @@ func FindReleaseByVersion( searchVersion := strings.TrimPrefix(version, "v") if strings.EqualFold(tagVersion, searchVersion) { - tagURL := fmt.Sprintf( - "https://github.com/%s/%s/tree/%s", - repo.Owner, - repo.Repo, - tag.Name, - ) return &VersionInfo{ Version: tag.Name, CommitSHA: tag.Commit.SHA, Source: "github_tag", CreatedAt: time.Time{}, // Tags don't have creation time in the API - URL: tagURL, + URL: fmt.Sprintf("https://github.com/%s/%s/commit/%s", repo.Owner, repo.Repo, tag.Commit.SHA), }, nil } } diff --git a/pkg/versioncommitfinder/versioncommitfinder.go b/pkg/versioncommitfinder/versioncommitfinder.go index a3ba2c77..82ddd0a5 100644 --- a/pkg/versioncommitfinder/versioncommitfinder.go +++ b/pkg/versioncommitfinder/versioncommitfinder.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/grafana/plugin-validator/pkg/grafana" + "github.com/grafana/plugin-validator/pkg/llmclient" "github.com/grafana/plugin-validator/pkg/logme" "github.com/grafana/plugin-validator/pkg/repotool" "github.com/grafana/plugin-validator/pkg/utils" @@ -27,6 +28,11 @@ type PackageJson struct { Version string `json:"version"` } +type geminiOutput struct { + CommitSHA string `json:"commitSHA"` + Reasoning string `json:"reasoning"` +} + // resolveVersion resolves the %VERSION% placeholder by reading package.json func resolveVersion(archivePath, pluginVersion string) (string, error) { if pluginVersion != "%VERSION%" { @@ -80,21 +86,6 @@ func FindPluginVersionsRefs( cleanup = cleanupFn } - pluginMetadata, err := utils.GetPluginMetadata(archivePath) - if err != nil { - logme.DebugFln("Failed to extract plugin metadata: %v", err) - return nil, nil, fmt.Errorf("failed to extract plugin metadata from archive: %w", err) - } - pluginID := pluginMetadata.ID - rawPluginVersion := pluginMetadata.Info.Version - - // Resolve %VERSION% placeholder if present - pluginVersion, err := resolveVersion(archivePath, rawPluginVersion) - if err != nil { - logme.DebugFln("Failed to resolve plugin version: %v", err) - return nil, nil, fmt.Errorf("failed to resolve plugin version: %w", err) - } - repoInfo, err := repotool.ParseRepoFromGitURL(githubURL) if err != nil { logme.DebugFln("Failed to parse GitHub URL: %v", err) @@ -107,6 +98,25 @@ func FindPluginVersionsRefs( repoInfo.Ref, ) + logme.DebugFln("Fetching repository tags") + // Fetch all tags so we can checkout to any tag ref (shallow clones don't include all tags) + fetchTagsCmd := exec.Command("git", "fetch", "--tags") + fetchTagsCmd.Dir = archivePath + if err := fetchTagsCmd.Run(); err != nil { + logme.DebugFln("Failed to fetch tags: %v", err) + } + + // Try to fetch the specific ref in case it's a branch (shallow clones only have default branch) + if repoInfo.Ref != "" { + logme.DebugFln("Fetching ref: %s", repoInfo.Ref) + fetchRefCmd := exec.Command("git", "fetch", "origin", repoInfo.Ref) + fetchRefCmd.Dir = archivePath + if err := fetchRefCmd.Run(); err != nil { + logme.DebugFln("Failed to fetch ref %s (may not be a branch): %v", repoInfo.Ref, err) + } + } + + logme.DebugFln("Checking out ref: %s", repoInfo.Ref) if repoInfo.Ref != "" { cmd := exec.Command("git", "checkout", repoInfo.Ref) cmd.Dir = archivePath @@ -128,12 +138,28 @@ func FindPluginVersionsRefs( } } + pluginMetadata, err := utils.GetPluginMetadata(archivePath) + if err != nil { + logme.DebugFln("Failed to extract plugin metadata: %v", err) + return nil, nil, fmt.Errorf("failed to extract plugin metadata from archive: %w", err) + } + pluginID := pluginMetadata.ID + rawPluginVersion := pluginMetadata.Info.Version + + // Resolve %VERSION% placeholder if present + pluginVersion, err := resolveVersion(archivePath, rawPluginVersion) + if err != nil { + logme.DebugFln("Failed to resolve plugin version: %v", err) + return nil, nil, fmt.Errorf("failed to resolve plugin version: %w", err) + } + var currentGrafanaVersion *repotool.VersionInfo grafanaClient := grafana.NewClient() grafanaVersions, err := grafanaClient.FindPluginVersions(pluginID) if err == nil && len(grafanaVersions) >= 1 { + logme.DebugFln("Found %d Grafana API versions", len(grafanaVersions)) grafanaAPIVersion := grafanaVersions[0] logme.DebugFln("Found Grafana API version: %s", grafanaAPIVersion.Version) @@ -156,6 +182,26 @@ func FindPluginVersionsRefs( } } } + logme.DebugFln("Current Grafana version: %s", currentGrafanaVersion) + + // If we still don't have a commit SHA, try gemini as a last resort + if currentGrafanaVersion != nil && currentGrafanaVersion.CommitSHA == "" { + logme.DebugFln("Attempting to find commit SHA using gemini CLI") + if sha, err := findCommitWithGemini(archivePath, currentGrafanaVersion.Version); err == nil && + sha != "" { + currentGrafanaVersion.CommitSHA = sha + currentGrafanaVersion.Source = "gemini" + currentGrafanaVersion.URL = fmt.Sprintf( + "https://github.com/%s/%s/commit/%s", + repoInfo.Owner, + repoInfo.Repo, + sha, + ) + logme.DebugFln("Found commit SHA via gemini: %s", sha) + } else if err != nil { + logme.DebugFln("Gemini fallback failed: %v", err) + } + } // Get current commit SHA for the submitted version cmd := exec.Command("git", "rev-parse", "HEAD") @@ -192,3 +238,132 @@ func FindPluginVersionsRefs( RepositoryPath: archivePath, }, cleanup, nil } + +// findCommitWithGemini uses gemini CLI as a last resort to find the commit SHA +// that corresponds to a specific version by analyzing git history. +func findCommitWithGemini(archivePath, version string) (string, error) { + client := llmclient.NewGeminiClient() + if err := client.CanUseLLM(); err != nil { + return "", fmt.Errorf("cannot use LLM: %w", err) + } + + logme.DebugFln("Using gemini CLI to find commit for version %s", version) + + // Build the prompt + prompt := fmt.Sprintf( + `You are in a git repository. Find the commit SHA for when version "%s was released". + +IMPORTANT: Follow this EXACT process step by step. Explain what you are doing and what you find at each step. + +## STEP 1: INVESTIGATION (find candidates) +Search for the version using multiple approaches: +- Check git tags: git tag -l "*%s*" or git tag -l +- Search commit messages: git log --oneline --grep="%s" +- Look for release patterns: git log --oneline --grep="release" +- Check recent history: git log --oneline -30 + +Different developers have different workflows, you must keep in consideration: +- Some bump the version first, then work on features, then package/release later +- Some implement changes first, then bump version at the end as the release commit +- CHANGELOGS are unreliable, always validate your findings + +We want the commit that "truly" signifies when the version was "completed" + +## STEP 2: VERIFICATION (mandatory - do not skip!) +Once you find a candidate commit, you MUST verify it by checking the actual SOURCE files: + +git show :src/plugin.json | grep version +git show :package.json | grep version + +IMPORTANT: IGNORE the dist/ folder - it contains build artifacts, not source files. + +Check src/plugin.json as the SOURCE OF TRUTH for the version. +- If src/plugin.json has an actual version number, that version MUST match "%s" +- If src/plugin.json has a "%%VERSION%%" placeholder, THEN check package.json for the actual version +- package.json is sometimes bumped BEFORE src/plugin.json - if they differ, keep looking for when src/plugin.json was updated to "%s" + +## STEP 3: DOUBLE-CHECK (mandatory - do not skip!) +Even if you found a matching commit, investigate AT LEAST ONE more approach: +- If you found it via tag, also check commit messages +- If you found it via commit message, also check tags +- Look at the commit BEFORE your candidate to see if version was already there +- Is this the commit that had a "full" version? Developers can bump version and keep pilling up commits after for the same version. + +This helps catch cases where version was bumped earlier than the "release" commit. + +## STEP 4: OUTPUT +Only after completing steps 1-3, write to output.json: + +{ + "commitSHA": "full 40-character SHA (use 'git rev-parse ' to expand if needed), or empty string if not found", + "reasoning": "brief explanation including: how you found it, what verification you did, what double-check you performed? Is it the "complete" version commit" +} + +IMPORTANT: The commitSHA MUST be the full 40-character hash, not a short hash. + +YOU MUST validate your JSON by running: jq . output.json +If it fails, fix the JSON and try again. + +Once output.json is valid, you are DONE. Exit immediately.`, + + version, + version, + version, + version, + version, + ) + + llmclient.CleanUpPromptFiles(archivePath) + + if err := client.CallLLM(prompt, archivePath, &llmclient.CallLLMOptions{ + // we are using gemini 2.5 flash after testing it against gemini 3 flash + // and see much better performance in speed and obeying instructions for this + // particular case + Model: "gemini-2.5-flash", + }); err != nil { + os.Remove(filepath.Join(archivePath, "output.json")) + return "", fmt.Errorf("gemini CLI failed: %w", err) + } + + // Read output.json + outputPath := filepath.Join(archivePath, "output.json") + outputData, err := os.ReadFile(outputPath) + if err != nil { + os.Remove(outputPath) + return "", fmt.Errorf("failed to read gemini output: %w", err) + } + + // Log the raw output for debugging + logme.DebugFln("Gemini output.json content: %s", string(outputData)) + + // Parse the output + var output geminiOutput + if err := json.Unmarshal(outputData, &output); err != nil { + os.Remove(outputPath) + return "", fmt.Errorf("failed to parse gemini output: %w", err) + } + + logme.DebugFln("Gemini found commit %s: %s", output.CommitSHA, output.Reasoning) + + // Cleanup + os.Remove(outputPath) + + // If we got a short SHA, try to expand it + commitSHA := output.CommitSHA + if len(commitSHA) > 0 && len(commitSHA) < 40 { + logme.DebugFln("Got short SHA %s, expanding with git rev-parse", commitSHA) + expandCmd := exec.Command("git", "rev-parse", commitSHA) + expandCmd.Dir = archivePath + if expandedOutput, err := expandCmd.Output(); err == nil { + commitSHA = strings.TrimSpace(string(expandedOutput)) + logme.DebugFln("Expanded to full SHA: %s", commitSHA) + } + } + + // Validate the commit SHA (should be 40 hex characters) + if len(commitSHA) != 40 { + return "", fmt.Errorf("invalid commit SHA length: %d", len(commitSHA)) + } + + return commitSHA, nil +}