diff --git a/README.md b/README.md index 705c0f0a..2f3403cb 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,7 @@ Run "mage gen:readme" to regenerate this section. | Discoverability / `discoverability` | Warns about missing keywords and description that are used for plugin indexing in the catalog. | None | | Go Manifest / `go-manifest` | Validates the build manifest. | None | | Go Security Checker / `go-sec` | Inspects source code for security problems by scanning the Go AST. | [gosec](https://github.com/securego/gosec), `sourceCodeUri` | +| Grafana Tooling Compliance / `toolingcompliance` | Ensures the plugin uses Grafana's standard plugin tooling (create-plugin). | None | | JS Source Map / `jsMap` | Checks for required `module.js.map` file(s) in archive. | `sourceCodeUri` | | Legacy Grafana Toolkit usage / `legacybuilder` | Detects the usage of the not longer supported Grafana Toolkit. | None | | Legacy Platform / `legacyplatform` | Detects use of Angular which is deprecated. | None | @@ -289,6 +290,7 @@ Run "mage gen:readme" to regenerate this section. | Safe Links / `safelinks` | Checks that links from `plugin.json` are safe. | None | | Screenshots / `screenshots` | Screenshots are specified in `plugin.json` that will be used in the Grafana plugin catalog. | None | | SDK Usage / `sdkusage` | Ensures that `grafana-plugin-sdk-go` is up-to-date. | None | +| SemVer Compliance / `semvercheck` | Uses LLM to detect breaking changes and verify version increments match SemVer conventions. | None | | Signature / `signature` | Ensures the plugin has a valid signature. | None | | Source Code / `sourcecode` | A comparison is made between the zip file and the source code to ensure what is released matches the repo associated with it. | `sourceCodeUri` | | Sponsorship Link / `sponsorshiplink` | Checks if a sponsorship link is specified in `plugin.json` that will be shown in the Grafana plugin catalog for users to support the plugin developer. | None | diff --git a/pkg/analysis/passes/analysis.go b/pkg/analysis/passes/analysis.go index e020a6af..d520d59f 100644 --- a/pkg/analysis/passes/analysis.go +++ b/pkg/analysis/passes/analysis.go @@ -41,10 +41,12 @@ import ( "github.com/grafana/plugin-validator/pkg/analysis/passes/safelinks" "github.com/grafana/plugin-validator/pkg/analysis/passes/screenshots" "github.com/grafana/plugin-validator/pkg/analysis/passes/sdkusage" + "github.com/grafana/plugin-validator/pkg/analysis/passes/semvercheck" "github.com/grafana/plugin-validator/pkg/analysis/passes/signature" "github.com/grafana/plugin-validator/pkg/analysis/passes/sourcecode" "github.com/grafana/plugin-validator/pkg/analysis/passes/sponsorshiplink" "github.com/grafana/plugin-validator/pkg/analysis/passes/templatereadme" + "github.com/grafana/plugin-validator/pkg/analysis/passes/toolingcompliance" "github.com/grafana/plugin-validator/pkg/analysis/passes/trackingscripts" "github.com/grafana/plugin-validator/pkg/analysis/passes/typesuffix" "github.com/grafana/plugin-validator/pkg/analysis/passes/unsafesvg" @@ -91,9 +93,11 @@ var Analyzers = []*analysis.Analyzer{ restrictivedep.Analyzer, screenshots.Analyzer, sdkusage.Analyzer, + semvercheck.Analyzer, signature.Analyzer, sourcecode.Analyzer, templatereadme.Analyzer, + toolingcompliance.Analyzer, trackingscripts.Analyzer, typesuffix.Analyzer, unsafesvg.Analyzer, diff --git a/pkg/analysis/passes/semvercheck/prompt.txt b/pkg/analysis/passes/semvercheck/prompt.txt new file mode 100644 index 00000000..a3e074cb --- /dev/null +++ b/pkg/analysis/passes/semvercheck/prompt.txt @@ -0,0 +1,51 @@ +You will analyze changes in this Grafana plugin to detect breaking changes and verify SemVer compliance. Use `git diff` to see the diff and `git show` to see full files in a specific commit. + +Analyze the changes between the versions focusing on: +1. Breaking changes (API changes, removed functionality, behavioral changes that could break existing users) +2. New features (new functionality, new APIs, new options) +3. Bug fixes (corrections to existing functionality) + +## Definitions for Grafana Plugins + +### Breaking Changes (require MAJOR version bump) +- Removed or renamed exported functions, components, or types +- Changed function signatures (parameters, return types) +- Removed configuration options +- Changed default behavior in ways that could break existing setups +- Removed support for previously supported data formats +- Changed plugin.json schema in incompatible ways (removed fields, changed types) +- Removed panel options or changed their structure + +### New Features (require MINOR version bump) +- Added new exported functions, components, or types +- Added new configuration options +- Added new panel features or visualization options +- Added support for new data sources or formats +- Added new query capabilities + +### Bug Fixes (require PATCH version bump) +- Fixed incorrect behavior +- Fixed edge cases +- Performance improvements +- Documentation updates +- Dependency updates (non-breaking) + +New version: {{.NewVersion}} introduced in commit: {{.NewCommit}} +Current version: {{.CurrentVersion}} introduced in commit: {{.CurrentCommit}} + +Analyze the changes and write your analysis to a JSON file "replies.json" with this format: + +{ + "has_breaking_changes": true|false, + "has_new_features": true|false, + "has_bug_fixes": true|false, + "breaking_changes_list": ["description of breaking change 1", "description of breaking change 2"], + "new_features_list": ["description of new feature 1", "description of new feature 2"], + "bug_fixes_list": ["description of bug fix 1", "description of bug fix 2"], + "recommended_version_bump": "major|minor|patch", + "explanation": "Brief explanation of why this version bump is recommended based on the changes" +} + +Use jq to validate replies.json is valid JSON, fix any issues if necessary. + +Once you are finished say "I finished the SemVer analysis" diff --git a/pkg/analysis/passes/semvercheck/semvercheck.go b/pkg/analysis/passes/semvercheck/semvercheck.go new file mode 100644 index 00000000..9802fb15 --- /dev/null +++ b/pkg/analysis/passes/semvercheck/semvercheck.go @@ -0,0 +1,319 @@ +package semvercheck + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/grafana/plugin-validator/pkg/analysis" + "github.com/grafana/plugin-validator/pkg/analysis/passes/archive" + "github.com/grafana/plugin-validator/pkg/analysis/passes/metadata" + "github.com/grafana/plugin-validator/pkg/analysis/passes/metadatavalid" + "github.com/grafana/plugin-validator/pkg/analysis/passes/modulejs" + "github.com/grafana/plugin-validator/pkg/llmclient" + "github.com/grafana/plugin-validator/pkg/logme" + "github.com/grafana/plugin-validator/pkg/versioncommitfinder" + "github.com/hashicorp/go-version" +) + +//go:embed prompt.txt +var promptTemplate string + +// SemVerAnalysisResponse represents the LLM response for SemVer analysis +type SemVerAnalysisResponse struct { + HasBreakingChanges bool `json:"has_breaking_changes"` + HasNewFeatures bool `json:"has_new_features"` + HasBugFixes bool `json:"has_bug_fixes"` + BreakingChangesList []string `json:"breaking_changes_list"` + NewFeaturesList []string `json:"new_features_list"` + BugFixesList []string `json:"bug_fixes_list"` + RecommendedVersionBump string `json:"recommended_version_bump"` + Explanation string `json:"explanation"` +} + +var ( + semverMismatch = &analysis.Rule{ + Name: "semver-mismatch", + Severity: analysis.SuspectedProblem, + } + breakingChangeDetected = &analysis.Rule{ + Name: "breaking-change-detected", + Severity: analysis.SuspectedProblem, + } +) + +// blockingAnalyzers contains validators that, if they report errors, should cause +// the SemVer analysis to be skipped +var blockingAnalyzers = []*analysis.Analyzer{ + archive.Analyzer, + metadata.Analyzer, + metadatavalid.Analyzer, + modulejs.Analyzer, +} + +var Analyzer = &analysis.Analyzer{ + Name: "semvercheck", + Requires: blockingAnalyzers, + Run: run, + Rules: []*analysis.Rule{semverMismatch, breakingChangeDetected}, + ReadmeInfo: analysis.ReadmeInfo{ + Name: "SemVer Compliance", + Description: "Uses LLM to detect breaking changes and verify version increments match SemVer conventions.", + }, +} + +var llmClient llmclient.LLMClient + +func SetLLMClient(client llmclient.LLMClient) { + llmClient = client +} + +func init() { + llmClient = llmclient.NewGeminiClient() +} + +func isGitHubURL(url string) bool { + return strings.Contains(strings.ToLower(url), "github.com") +} + +func run(pass *analysis.Pass) (any, error) { + // Check if any blocking analyzers reported errors - silently skip to avoid noise + for _, analyzer := range blockingAnalyzers { + if pass.AnalyzerHasErrors(analyzer) { + return nil, nil + } + } + + if pass.CheckParams.SourceCodeReference == "" { + return nil, nil + } + + if os.Getenv("SKIP_LLM_SEMVER") != "" { + return nil, nil + } + + if err := llmClient.CanUseLLM(); err != nil { + return nil, nil + } + + // Only support GitHub URLs + if !isGitHubURL(pass.CheckParams.SourceCodeReference) { + logme.Debugln( + "Source code reference is not a GitHub URL:", + pass.CheckParams.SourceCodeReference, + ) + return nil, nil + } + + versions, cleanup, err := versioncommitfinder.FindPluginVersionsRefs( + pass.CheckParams.SourceCodeReference, + "", + ) + if err != nil { + logme.Debugln("Failed to find versions", err) + return nil, nil + } + defer cleanup() + + // Need both versions to compare + if versions.CurrentGrafanaVersion == nil || versions.SubmittedGitHubVersion == nil || + versions.CurrentGrafanaVersion.CommitSHA == "" || versions.SubmittedGitHubVersion.CommitSHA == "" { + logme.Debugln("Cannot run SemVer analysis - missing version information") + return nil, nil + } + + // Parse versions + currentVersion, err := version.NewVersion(versions.CurrentGrafanaVersion.Version) + if err != nil { + logme.Debugln("Failed to parse current version:", err) + return nil, nil + } + + newVersion, err := version.NewVersion(versions.SubmittedGitHubVersion.Version) + if err != nil { + logme.Debugln("Failed to parse new version:", err) + return nil, nil + } + + // Determine version bump type + versionBumpType := determineVersionBumpType(currentVersion, newVersion) + + // Run LLM analysis to detect changes + response, err := runSemVerLLMAnalysis( + versions.SubmittedGitHubVersion.Version, + versions.SubmittedGitHubVersion.CommitSHA, + versions.CurrentGrafanaVersion.Version, + versions.CurrentGrafanaVersion.CommitSHA, + versions.RepositoryPath, + ) + if err != nil { + logme.Debugln("Failed to run SemVer LLM analysis:", err) + return nil, nil + } + + // Report breaking changes + if response.HasBreakingChanges { + breakingChangesDetail := formatBreakingChanges(response.BreakingChangesList) + pass.ReportResult( + pass.AnalyzerName, + breakingChangeDetected, + fmt.Sprintf("Breaking changes detected in version %s → %s", + versions.CurrentGrafanaVersion.Version, + versions.SubmittedGitHubVersion.Version), + breakingChangesDetail, + ) + } + + // Check for SemVer mismatch + if response.HasBreakingChanges && versionBumpType != "major" { + pass.ReportResult( + pass.AnalyzerName, + semverMismatch, + "SemVer mismatch: Breaking changes require major version bump", + fmt.Sprintf( + "Breaking changes were detected, but the version was only bumped from %s to %s (%s bump). "+ + "According to SemVer, breaking changes require a major version bump (e.g., 1.x.x → 2.0.0).\n\n"+ + "**Detected breaking changes:**\n%s\n\n"+ + "**Recommendation:** %s", + versions.CurrentGrafanaVersion.Version, + versions.SubmittedGitHubVersion.Version, + versionBumpType, + formatBreakingChanges(response.BreakingChangesList), + response.Explanation, + ), + ) + } else if response.HasNewFeatures && !response.HasBreakingChanges && versionBumpType == "patch" { + pass.ReportResult( + pass.AnalyzerName, + semverMismatch, + "SemVer mismatch: New features typically require minor version bump", + fmt.Sprintf( + "New features were detected, but the version was only bumped from %s to %s (patch bump). "+ + "According to SemVer, new features typically require a minor version bump (e.g., 1.0.x → 1.1.0).\n\n"+ + "**Detected new features:**\n%s\n\n"+ + "**Recommendation:** %s", + versions.CurrentGrafanaVersion.Version, + versions.SubmittedGitHubVersion.Version, + formatFeaturesList(response.NewFeaturesList), + response.Explanation, + ), + ) + } + + return response, nil +} + +func determineVersionBumpType(current, new *version.Version) string { + currentSegments := current.Segments() + newSegments := new.Segments() + + // Ensure we have at least 3 segments + for len(currentSegments) < 3 { + currentSegments = append(currentSegments, 0) + } + for len(newSegments) < 3 { + newSegments = append(newSegments, 0) + } + + if newSegments[0] > currentSegments[0] { + return "major" + } + if newSegments[1] > currentSegments[1] { + return "minor" + } + if newSegments[2] > currentSegments[2] { + return "patch" + } + + return "none" +} + +func formatBreakingChanges(changes []string) string { + if len(changes) == 0 { + return "No specific breaking changes identified." + } + var formatted []string + for _, change := range changes { + formatted = append(formatted, "- "+change) + } + return strings.Join(formatted, "\n") +} + +func formatFeaturesList(features []string) string { + if len(features) == 0 { + return "No specific new features identified." + } + var formatted []string + for _, feature := range features { + formatted = append(formatted, "- "+feature) + } + return strings.Join(formatted, "\n") +} + +func generatePrompt(newVersion, newCommit, currentVersion, currentCommit string) (string, error) { + if newVersion == "" || newCommit == "" || currentVersion == "" || currentCommit == "" { + return "", fmt.Errorf("version information incomplete") + } + + tmpl, err := template.New("prompt").Parse(promptTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse prompt template: %w", err) + } + + data := map[string]any{ + "NewVersion": newVersion, + "NewCommit": newCommit, + "CurrentVersion": currentVersion, + "CurrentCommit": currentCommit, + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("failed to execute prompt template: %w", err) + } + + return buf.String(), nil +} + +func runSemVerLLMAnalysis( + newVersion, newCommit, currentVersion, currentCommit, repositoryPath string, +) (*SemVerAnalysisResponse, error) { + prompt, err := generatePrompt(newVersion, newCommit, currentVersion, currentCommit) + if err != nil { + logme.Debugln("Failed to generate prompt:", err) + return nil, err + } + + llmclient.CleanUpPromptFiles(repositoryPath) + + if err := llmClient.CallLLM(prompt, repositoryPath, nil); err != nil { + logme.Debugln("Failed to call LLM:", err) + return nil, err + } + + responsesPath := filepath.Join(repositoryPath, "replies.json") + if _, err := os.Stat(responsesPath); err != nil { + logme.Debugln("replies.json file not found:", err) + return nil, fmt.Errorf("replies.json file not found: %w", err) + } + + responsesData, err := os.ReadFile(responsesPath) + if err != nil { + logme.Debugln("Failed to read replies.json:", err) + return nil, fmt.Errorf("failed to read replies.json: %w", err) + } + + var response SemVerAnalysisResponse + if err := json.Unmarshal(responsesData, &response); err != nil { + logme.Debugln("Failed to parse replies.json:", err) + return nil, fmt.Errorf("failed to parse replies.json: %w", err) + } + + logme.Debugln("SemVer LLM analysis completed successfully") + return &response, nil +} diff --git a/pkg/analysis/passes/semvercheck/semvercheck_test.go b/pkg/analysis/passes/semvercheck/semvercheck_test.go new file mode 100644 index 00000000..438c7e47 --- /dev/null +++ b/pkg/analysis/passes/semvercheck/semvercheck_test.go @@ -0,0 +1,177 @@ +package semvercheck + +import ( + "testing" + + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/require" +) + +func TestDetermineVersionBumpType(t *testing.T) { + tests := []struct { + name string + current string + new string + expected string + }{ + { + name: "major bump", + current: "1.2.3", + new: "2.0.0", + expected: "major", + }, + { + name: "minor bump", + current: "1.2.3", + new: "1.3.0", + expected: "minor", + }, + { + name: "patch bump", + current: "1.2.3", + new: "1.2.4", + expected: "patch", + }, + { + name: "major bump from 0.x", + current: "0.9.9", + new: "1.0.0", + expected: "major", + }, + { + name: "no change", + current: "1.2.3", + new: "1.2.3", + expected: "none", + }, + { + name: "short version numbers", + current: "1.2", + new: "1.3", + expected: "minor", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + current, err := version.NewVersion(tt.current) + require.NoError(t, err) + new, err := version.NewVersion(tt.new) + require.NoError(t, err) + + result := determineVersionBumpType(current, new) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestFormatBreakingChanges(t *testing.T) { + tests := []struct { + name string + changes []string + expected string + }{ + { + name: "empty list", + changes: []string{}, + expected: "No specific breaking changes identified.", + }, + { + name: "single change", + changes: []string{"Removed deprecated API"}, + expected: "- Removed deprecated API", + }, + { + name: "multiple changes", + changes: []string{"Removed deprecated API", "Changed function signature"}, + expected: "- Removed deprecated API\n- Changed function signature", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := formatBreakingChanges(tt.changes) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestFormatFeaturesList(t *testing.T) { + tests := []struct { + name string + features []string + expected string + }{ + { + name: "empty list", + features: []string{}, + expected: "No specific new features identified.", + }, + { + name: "single feature", + features: []string{"Added new panel option"}, + expected: "- Added new panel option", + }, + { + name: "multiple features", + features: []string{"Added new panel option", "Added query caching"}, + expected: "- Added new panel option\n- Added query caching", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := formatFeaturesList(tt.features) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestGeneratePrompt(t *testing.T) { + tests := []struct { + name string + newVersion string + newCommit string + currentVersion string + currentCommit string + expectError bool + }{ + { + name: "valid inputs", + newVersion: "2.0.0", + newCommit: "abc123", + currentVersion: "1.0.0", + currentCommit: "def456", + expectError: false, + }, + { + name: "empty new version", + newVersion: "", + newCommit: "abc123", + currentVersion: "1.0.0", + currentCommit: "def456", + expectError: true, + }, + { + name: "empty commit", + newVersion: "2.0.0", + newCommit: "", + currentVersion: "1.0.0", + currentCommit: "def456", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := generatePrompt(tt.newVersion, tt.newCommit, tt.currentVersion, tt.currentCommit) + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Contains(t, result, tt.newVersion) + require.Contains(t, result, tt.currentVersion) + } + }) + } +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/.config/.gitkeep b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/.config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/package.json b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/package.json new file mode 100644 index 00000000..3dc4736c --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/package.json @@ -0,0 +1,13 @@ +{ + "name": "custom-webpack-plugin", + "version": "1.0.0", + "scripts": { + "dev": "webpack -w", + "build": "webpack", + "test": "jest", + "lint": "eslint ." + }, + "devDependencies": { + "@grafana/create-plugin": "^4.0.0" + } +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/tsconfig.json b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/tsconfig.json new file mode 100644 index 00000000..102e45b7 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./.config/tsconfig.json" +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/webpack.config.ts b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/webpack.config.ts new file mode 100644 index 00000000..d6189354 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/custom-webpack/webpack.config.ts @@ -0,0 +1,8 @@ +// Custom webpack config that doesn't use .config +const path = require('path'); +module.exports = { + entry: './src/index.ts', + output: { + path: path.resolve(__dirname, 'dist'), + } +}; diff --git a/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/.config/.gitkeep b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/.config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/.config/webpack/webpack.config.ts b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/.config/webpack/webpack.config.ts new file mode 100644 index 00000000..ff8b4c56 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/.config/webpack/webpack.config.ts @@ -0,0 +1 @@ +export default {}; diff --git a/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/package.json b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/package.json new file mode 100644 index 00000000..fed30b1b --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/package.json @@ -0,0 +1,15 @@ +{ + "name": "fully-compliant-plugin", + "version": "1.0.0", + "scripts": { + "dev": "webpack -w -c ./.config/webpack/webpack.config.ts --env development", + "build": "webpack -c ./.config/webpack/webpack.config.ts --env production", + "test": "jest --passWithNoTests", + "lint": "eslint --cache --ignore-path ./.gitignore --ext .js,.jsx,.ts,.tsx ." + }, + "devDependencies": { + "@grafana/create-plugin": "^4.0.0", + "@grafana/eslint-config": "^7.0.0", + "@grafana/tsconfig": "^1.3.0" + } +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/tsconfig.json b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/tsconfig.json new file mode 100644 index 00000000..102e45b7 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./.config/tsconfig.json" +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/webpack.config.ts b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/webpack.config.ts new file mode 100644 index 00000000..1f9aa554 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/fully-compliant/webpack.config.ts @@ -0,0 +1,2 @@ +import config from './.config/webpack/webpack.config.ts'; +export default config; diff --git a/pkg/analysis/passes/toolingcompliance/testdata/has-tooling-no-config/package.json b/pkg/analysis/passes/toolingcompliance/testdata/has-tooling-no-config/package.json new file mode 100644 index 00000000..041e90c6 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/has-tooling-no-config/package.json @@ -0,0 +1,7 @@ +{ + "name": "has-tooling-no-config-plugin", + "version": "1.0.0", + "devDependencies": { + "@grafana/create-plugin": "^4.0.0" + } +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/missing-config/package.json b/pkg/analysis/passes/toolingcompliance/testdata/missing-config/package.json new file mode 100644 index 00000000..a768751b --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/missing-config/package.json @@ -0,0 +1,7 @@ +{ + "name": "missing-config-plugin", + "version": "1.0.0", + "devDependencies": { + "webpack": "^5.0.0" + } +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/.config/.gitkeep b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/.config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/package.json b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/package.json new file mode 100644 index 00000000..c4764cc8 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/package.json @@ -0,0 +1,11 @@ +{ + "name": "missing-scripts-plugin", + "version": "1.0.0", + "scripts": { + "dev": "webpack -w", + "build": "webpack" + }, + "devDependencies": { + "@grafana/create-plugin": "^4.0.0" + } +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/tsconfig.json b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/tsconfig.json new file mode 100644 index 00000000..102e45b7 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./.config/tsconfig.json" +} diff --git a/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/webpack.config.ts b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/webpack.config.ts new file mode 100644 index 00000000..1f9aa554 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/testdata/missing-scripts/webpack.config.ts @@ -0,0 +1,2 @@ +import config from './.config/webpack/webpack.config.ts'; +export default config; diff --git a/pkg/analysis/passes/toolingcompliance/testdata/no-source/.keepempty b/pkg/analysis/passes/toolingcompliance/testdata/no-source/.keepempty new file mode 100644 index 00000000..e69de29b diff --git a/pkg/analysis/passes/toolingcompliance/toolingcompliance.go b/pkg/analysis/passes/toolingcompliance/toolingcompliance.go new file mode 100644 index 00000000..a8f68295 --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/toolingcompliance.go @@ -0,0 +1,258 @@ +package toolingcompliance + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/grafana/plugin-validator/pkg/analysis" + "github.com/grafana/plugin-validator/pkg/analysis/passes/published" + "github.com/grafana/plugin-validator/pkg/analysis/passes/sourcecode" +) + +var ( + missingConfigDir = &analysis.Rule{ + Name: "missing-config-dir", + Severity: analysis.Error, + } + missingGrafanaTooling = &analysis.Rule{ + Name: "missing-grafana-tooling", + Severity: analysis.Error, + } + invalidWebpackConfig = &analysis.Rule{ + Name: "invalid-webpack-config", + Severity: analysis.Warning, + } + invalidTsConfig = &analysis.Rule{ + Name: "invalid-tsconfig", + Severity: analysis.Warning, + } + missingStandardScripts = &analysis.Rule{ + Name: "missing-standard-scripts", + Severity: analysis.Warning, + } +) + +var Analyzer = &analysis.Analyzer{ + Name: "toolingcompliance", + Requires: []*analysis.Analyzer{sourcecode.Analyzer, published.Analyzer}, + Run: run, + Rules: []*analysis.Rule{ + missingConfigDir, + missingGrafanaTooling, + invalidWebpackConfig, + invalidTsConfig, + missingStandardScripts, + }, + ReadmeInfo: analysis.ReadmeInfo{ + Name: "Grafana Tooling Compliance", + Description: "Ensures the plugin uses Grafana's standard plugin tooling (create-plugin).", + }, +} + +// ToolingCheck represents the result of the tooling compliance check +type ToolingCheck struct { + HasConfigDir bool + HasGrafanaTooling bool + HasValidWebpackConfig bool + HasValidTsConfig bool + HasStandardScripts bool + MissingScripts []string + ToolingDeviationScore int // 0 = fully compliant, higher = more deviation +} + +// Standard scripts expected in a create-plugin project +var standardScripts = []string{"dev", "build", "test", "lint"} + +func run(pass *analysis.Pass) (interface{}, error) { + sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + + // If no source code directory is provided, we can't check tooling compliance + if !ok || sourceCodeDir == "" { + return nil, nil + } + + result := &ToolingCheck{} + + // Get published status to adjust severity for existing plugins + publishedStatus, ok := pass.ResultOf[published.Analyzer].(*published.PluginStatus) + isPublished := ok && publishedStatus.Status != "unknown" + + // If the plugin is already published, reduce severity to warning + if isPublished { + missingConfigDir.Severity = analysis.Warning + missingGrafanaTooling.Severity = analysis.Warning + } + + // Check 1: .config directory presence + configDir := filepath.Join(sourceCodeDir, ".config") + if _, err := os.Stat(configDir); err == nil { + result.HasConfigDir = true + } else { + result.ToolingDeviationScore += 3 + pass.ReportResult( + pass.AnalyzerName, + missingConfigDir, + "Missing .config directory", + "The plugin source code is missing the .config directory. This indicates the plugin was not created using Grafana's create-plugin tool. Please use https://grafana.com/developers/plugin-tools/ to create and maintain your plugin.", + ) + } + + // Check 2: Grafana tooling packages in package.json + packageJsonPath := filepath.Join(sourceCodeDir, "package.json") + packageJson, err := parsePackageJson(packageJsonPath) + if err == nil { + result.HasGrafanaTooling = checkGrafanaToolingPackages(packageJson) + if !result.HasGrafanaTooling { + result.ToolingDeviationScore += 3 + pass.ReportResult( + pass.AnalyzerName, + missingGrafanaTooling, + "Plugin not using Grafana plugin tooling", + "The plugin's package.json does not include @grafana/create-plugin or related tooling packages in devDependencies. Plugins should be built using Grafana's official tooling. Please see https://grafana.com/developers/plugin-tools/get-started/set-up-development-environment for setup instructions.", + ) + } + + // Check 3: Standard scripts in package.json + result.MissingScripts = checkStandardScripts(packageJson) + if len(result.MissingScripts) == 0 { + result.HasStandardScripts = true + } else { + result.ToolingDeviationScore++ + pass.ReportResult( + pass.AnalyzerName, + missingStandardScripts, + "Missing standard package.json scripts", + "The plugin's package.json is missing some standard scripts: "+strings.Join(result.MissingScripts, ", ")+". Plugins created with create-plugin include scripts for dev, build, test, and lint. See https://grafana.com/developers/plugin-tools/ for more information.", + ) + } + } + + // Check 4: webpack.config.ts extends from .config + result.HasValidWebpackConfig = checkWebpackConfig(sourceCodeDir) + if !result.HasValidWebpackConfig && result.HasConfigDir { + // Only report if .config exists but webpack doesn't extend from it + result.ToolingDeviationScore++ + pass.ReportResult( + pass.AnalyzerName, + invalidWebpackConfig, + "webpack.config.ts does not extend from .config", + "The plugin has a .config directory but webpack.config.ts does not import from './.config/webpack/webpack.config.ts'. This indicates the build configuration may not be using Grafana's standard tooling. See https://grafana.com/developers/plugin-tools/ for the expected configuration.", + ) + } + + // Check 5: tsconfig.json extends from .config + result.HasValidTsConfig = checkTsConfig(sourceCodeDir) + if !result.HasValidTsConfig && result.HasConfigDir { + // Only report if .config exists but tsconfig doesn't extend from it + result.ToolingDeviationScore++ + pass.ReportResult( + pass.AnalyzerName, + invalidTsConfig, + "tsconfig.json does not extend from .config", + "The plugin has a .config directory but tsconfig.json does not extend from './.config/tsconfig.json'. This indicates the TypeScript configuration may not be using Grafana's standard tooling. See https://grafana.com/developers/plugin-tools/ for the expected configuration.", + ) + } + + return result, nil +} + +// PackageJsonFull represents the full package.json structure we need +type PackageJsonFull struct { + Name string `json:"name"` + Version string `json:"version"` + Scripts map[string]string `json:"scripts"` + DevDependencies map[string]string `json:"devDependencies"` + Dependencies map[string]string `json:"dependencies"` +} + +func parsePackageJson(path string) (*PackageJsonFull, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var packageJson PackageJsonFull + if err := json.Unmarshal(data, &packageJson); err != nil { + return nil, err + } + + return &packageJson, nil +} + +func checkGrafanaToolingPackages(packageJson *PackageJsonFull) bool { + // List of Grafana tooling packages that indicate proper tooling usage + grafanaToolingPackages := []string{ + "@grafana/create-plugin", + "@grafana/plugin-configs", + "@grafana/eslint-config", + "@grafana/tsconfig", + } + + // Check devDependencies for any of the Grafana tooling packages + for _, pkg := range grafanaToolingPackages { + if _, ok := packageJson.DevDependencies[pkg]; ok { + return true + } + // Also check regular dependencies as a fallback + if _, ok := packageJson.Dependencies[pkg]; ok { + return true + } + } + + return false +} + +func checkStandardScripts(packageJson *PackageJsonFull) []string { + var missing []string + for _, script := range standardScripts { + if _, ok := packageJson.Scripts[script]; !ok { + missing = append(missing, script) + } + } + return missing +} + +func checkWebpackConfig(sourceCodeDir string) bool { + // Check for webpack.config.ts or webpack.config.js + webpackConfigPaths := []string{ + filepath.Join(sourceCodeDir, "webpack.config.ts"), + filepath.Join(sourceCodeDir, "webpack.config.js"), + } + + for _, configPath := range webpackConfigPaths { + content, err := os.ReadFile(configPath) + if err != nil { + continue + } + + contentStr := string(content) + // Check if it imports from .config/webpack + if strings.Contains(contentStr, "./.config/webpack") || + strings.Contains(contentStr, ".config/webpack") || + strings.Contains(contentStr, "@grafana/plugin-configs") { + return true + } + } + + return false +} + +func checkTsConfig(sourceCodeDir string) bool { + tsconfigPath := filepath.Join(sourceCodeDir, "tsconfig.json") + content, err := os.ReadFile(tsconfigPath) + if err != nil { + return false + } + + contentStr := string(content) + // Check if it extends from .config/tsconfig.json or uses @grafana/tsconfig + if strings.Contains(contentStr, "./.config/tsconfig.json") || + strings.Contains(contentStr, ".config/tsconfig.json") || + strings.Contains(contentStr, "@grafana/tsconfig") { + return true + } + + return false +} diff --git a/pkg/analysis/passes/toolingcompliance/toolingcompliance_test.go b/pkg/analysis/passes/toolingcompliance/toolingcompliance_test.go new file mode 100644 index 00000000..2c05359b --- /dev/null +++ b/pkg/analysis/passes/toolingcompliance/toolingcompliance_test.go @@ -0,0 +1,214 @@ +package toolingcompliance + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/plugin-validator/pkg/analysis" + "github.com/grafana/plugin-validator/pkg/analysis/passes/published" + "github.com/grafana/plugin-validator/pkg/analysis/passes/sourcecode" + "github.com/grafana/plugin-validator/pkg/testpassinterceptor" +) + +func TestToolingComplianceFullyCompliant(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: filepath.Join("testdata", "fully-compliant"), + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.NotNil(t, result) + + toolingCheck := result.(*ToolingCheck) + require.True(t, toolingCheck.HasConfigDir) + require.True(t, toolingCheck.HasGrafanaTooling) + require.True(t, toolingCheck.HasValidWebpackConfig) + require.True(t, toolingCheck.HasValidTsConfig) + require.True(t, toolingCheck.HasStandardScripts) + require.Equal(t, 0, toolingCheck.ToolingDeviationScore) + require.Len(t, interceptor.Diagnostics, 0) +} + +func TestToolingComplianceMissingConfigDir(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: filepath.Join("testdata", "missing-config"), + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.NotNil(t, result) + + toolingCheck := result.(*ToolingCheck) + require.False(t, toolingCheck.HasConfigDir) + require.False(t, toolingCheck.HasGrafanaTooling) + + // Should have at least 2 diagnostics - missing config and missing tooling + require.GreaterOrEqual(t, len(interceptor.Diagnostics), 2) + + // Check that the diagnostics are errors for new plugins + hasConfigError := false + hasToolingError := false + for _, diag := range interceptor.Diagnostics { + if diag.Name == "missing-config-dir" { + hasConfigError = true + require.Equal(t, analysis.Error, diag.Severity) + } + if diag.Name == "missing-grafana-tooling" { + hasToolingError = true + require.Equal(t, analysis.Error, diag.Severity) + } + } + require.True(t, hasConfigError, "Should have missing-config-dir diagnostic") + require.True(t, hasToolingError, "Should have missing-grafana-tooling diagnostic") +} + +func TestToolingCompliancePublishedPluginGetsWarning(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: filepath.Join("testdata", "missing-config"), + published.Analyzer: &published.PluginStatus{ + Status: "published", + }, + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.NotNil(t, result) + + toolingCheck := result.(*ToolingCheck) + require.False(t, toolingCheck.HasConfigDir) + + // Should have diagnostics as warnings for published plugins + require.GreaterOrEqual(t, len(interceptor.Diagnostics), 2) + for _, diag := range interceptor.Diagnostics { + if diag.Name == "missing-config-dir" || diag.Name == "missing-grafana-tooling" { + require.Equal(t, analysis.Warning, diag.Severity) + } + } +} + +func TestToolingComplianceNoSourceCode(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: "", + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.Nil(t, result) + require.Len(t, interceptor.Diagnostics, 0) +} + +func TestToolingComplianceHasToolingButNoConfigDir(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: filepath.Join("testdata", "has-tooling-no-config"), + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.NotNil(t, result) + + toolingCheck := result.(*ToolingCheck) + require.False(t, toolingCheck.HasConfigDir) + require.True(t, toolingCheck.HasGrafanaTooling) + + // Should have at least 1 diagnostic for missing config dir + hasConfigError := false + for _, diag := range interceptor.Diagnostics { + if diag.Name == "missing-config-dir" { + hasConfigError = true + } + } + require.True(t, hasConfigError, "Should have missing-config-dir diagnostic") +} + +func TestToolingComplianceCustomWebpackConfig(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: filepath.Join("testdata", "custom-webpack"), + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.NotNil(t, result) + + toolingCheck := result.(*ToolingCheck) + require.True(t, toolingCheck.HasConfigDir) + require.False(t, toolingCheck.HasValidWebpackConfig) + + // Should have a diagnostic for invalid webpack config + hasWebpackWarning := false + for _, diag := range interceptor.Diagnostics { + if diag.Name == "invalid-webpack-config" { + hasWebpackWarning = true + require.Equal(t, analysis.Warning, diag.Severity) + } + } + require.True(t, hasWebpackWarning, "Should have invalid-webpack-config diagnostic") +} + +func TestToolingComplianceMissingScripts(t *testing.T) { + var interceptor testpassinterceptor.TestPassInterceptor + + pass := &analysis.Pass{ + RootDir: filepath.Join("./"), + ResultOf: map[*analysis.Analyzer]interface{}{ + sourcecode.Analyzer: filepath.Join("testdata", "missing-scripts"), + }, + Report: interceptor.ReportInterceptor(), + } + + result, err := Analyzer.Run(pass) + require.NoError(t, err) + require.NotNil(t, result) + + toolingCheck := result.(*ToolingCheck) + require.False(t, toolingCheck.HasStandardScripts) + require.Contains(t, toolingCheck.MissingScripts, "test") + require.Contains(t, toolingCheck.MissingScripts, "lint") + + // Should have a diagnostic for missing scripts + hasScriptsWarning := false + for _, diag := range interceptor.Diagnostics { + if diag.Name == "missing-standard-scripts" { + hasScriptsWarning = true + require.Equal(t, analysis.Warning, diag.Severity) + } + } + require.True(t, hasScriptsWarning, "Should have missing-standard-scripts diagnostic") +}