[feat]: PAC AI pipelinerun failure summary

pkg/oc/oc.go

@@ -172,4 +172,4 @@ func CopySecret(secretName string, sourceNamespace string, destNamespace string)
 	cmdOutput := cmd.MustSucceed("bash", "-c", fmt.Sprintf(`echo '%s' | jq 'del(.metadata["namespace", "creationTimestamp", "resourceVersion", "selfLink", "uid", "annotations"]) | .data |= with_entries(if .key == "github-auth-key" then .key = "token" else . end)'`, secretJson)).Stdout()
 	cmd.MustSucceed("bash", "-c", fmt.Sprintf(`echo '%s' | kubectl apply -n %s -f -`, cmdOutput, destNamespace))
 	log.Printf("Successfully copied secret %s from %s to %s", secretName, sourceNamespace, destNamespace)
-}
+}

pkg/pac/pac.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"log"
 	"math/big"
@@ -22,6 +23,7 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/git"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
 	"github.com/openshift-pipelines/release-tests/pkg/clients"
+	rtcmd "github.com/openshift-pipelines/release-tests/pkg/cmd"
 	"github.com/openshift-pipelines/release-tests/pkg/config"
 	"github.com/openshift-pipelines/release-tests/pkg/k8s"
 	"github.com/openshift-pipelines/release-tests/pkg/oc"
@@ -33,6 +35,7 @@ import (
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 const (
@@ -231,9 +234,110 @@ func createNewRepository(c *clients.Clients, projectName, targetGroupNamespace,
 	}
 
 	log.Printf("Repository %q created successfully in namespace %q", repo.GetName(), repo.GetNamespace())
+	// Store the Repository name so we can later patch it with AI settings.
+	store.PutScenarioData("pacRepositoryName", repo.GetName())
 	return nil
 }
 
+// ConfigureRepositoryAISettings patches the Pipelines-as-Code Repository resource
+// to enable AI/LLM analysis using a configuration similar to the example in the
+// release notes.
+func ConfigureRepositoryAISettings() {
+	repoName := store.GetScenarioData("pacRepositoryName")
+	if repoName == "" {
+		log.Printf("pacRepositoryName not set; skipping AI settings configuration on Repository")
+		return
+	}
+
+	namespace := store.Namespace()
+	c := store.Clients()
+
+	// Ensure the Gemini API key secret exists in the same namespace as the
+	// Repository resource, per docs:
+	// https://pipelinesascode.com/docs/guide/llm-analysis/#aillm-powered-pipeline-analysis
+	geminiToken := os.Getenv("GEMINI_API_KEY")
+	if geminiToken == "" {
+		testsuit.T.Fail(fmt.Errorf("GEMINI_API_KEY environment variable not set; cannot configure AI analysis"))
+		return
+	}
+	if !oc.SecretExists("gemini-api-key", namespace) {
+		log.Printf("Creating Gemini API key secret 'gemini-api-key' in namespace %q", namespace)
+		rtcmd.MustSucceed("oc", "create", "secret", "generic", "gemini-api-key", "--from-literal", "token="+geminiToken, "-n", namespace)
+	} else {
+		log.Printf("Gemini API key secret 'gemini-api-key' already exists in namespace %q", namespace)
+	}
+
+	// Timestamp checkpoint for validating that the AI comment is created/updated
+	// after we configured the feature.
+	store.PutScenarioData("aiCommentSince", time.Now().UTC().Format(time.RFC3339Nano))
+
+	model := os.Getenv("PAC_AI_MODEL") // Optional override. If empty, use provider default.
+
+	role := map[string]any{
+		"name": "gemini-failure-analysis",
+		"prompt": `Analyze this failed pipeline run from a Google Gemini perspective:
+
+					1. Root cause
+
+					2. Fix steps
+
+					3. Preventive measures
+
+					Start your response with the exact marker: PAC_AI_GEMINI_FAILURE_ANALYSIS`,
+		"on_cel": `body.pipelineRun.status.conditions[0].reason == "Failed"`,
+		"context_items": map[string]any{
+			"error_content": true,
+			"container_logs": map[string]any{
+				"enabled":   true,
+				"max_lines": 100,
+			},
+		},
+		"output": "pr-comment",
+	}
+	// If model is not provided, the framework uses the provider default
+	// (for Gemini: gemini-2.5-flash-lite).
+	if model != "" {
+		role["model"] = model
+	}
+
+	patchObj := map[string]any{
+		"spec": map[string]any{
+			"settings": map[string]any{
+				"ai": map[string]any{
+					"enabled":         true,
+					"provider":        "gemini",
+					"timeout_seconds": 30,
+					"max_tokens":      1000,
+					"secret_ref": map[string]any{
+						"name": "gemini-api-key",
+						"key":  "token",
+					},
+					"roles": []any{role},
+				},
+			},
+		},
+	}
+
+	patchBytes, err := json.Marshal(patchObj)
+	if err != nil {
+		testsuit.T.Fail(fmt.Errorf("failed to marshal AI settings patch for Repository %q: %v", repoName, err))
+		return
+	}
+
+	if _, err := c.PacClientset.Repositories(namespace).Patch(
+		context.Background(),
+		repoName,
+		types.MergePatchType,
+		patchBytes,
+		metav1.PatchOptions{},
+	); err != nil {
+		testsuit.T.Fail(fmt.Errorf("failed to patch Repository %q with AI settings: %v", repoName, err))
+		return
+	}
+
+	log.Printf("Configured AI analysis settings on Repository %q in namespace %q", repoName, namespace)
+}
+
 // addLabelToProject adds a label to a GitLab project
 func addLabelToProject(projectID int, labelName, color, description string) error {
 	// Check if the label already exists
@@ -819,3 +923,72 @@ func CleanupPAC(c *clients.Clients, smeeDeploymentName, namespace string) {
 		testsuit.T.Fail(fmt.Errorf("failed to Delete Smee Deployment: %v", err))
 	}
 }
+
+func MakePullRequestPipelineFail() {
+	data, err := os.ReadFile(config.Path("testdata", "pac", "pull-request-go-fail.yaml"))
+	if err != nil {
+		testsuit.T.Fail(fmt.Errorf("failed to read testdata failing pull_request PipelineRun: %v", err))
+	}
+
+	if err := validateYAML(data); err != nil {
+		testsuit.T.Fail(fmt.Errorf("invalid YAML in testdata failing pull_request PipelineRun: %v", err))
+	}
+
+	if err := os.WriteFile(pullRequestFileName, data, 0600); err != nil {
+		testsuit.T.Fail(fmt.Errorf("failed to write %s: %v", pullRequestFileName, err))
+	}
+	log.Printf("Wrote failing pull-request.yaml from testdata to %s", pullRequestFileName)
+}
+
+// Validates AI analysis comment in MR
+func ValidateAIMRComment() {
+	projectID, err := strconv.Atoi(store.GetScenarioData("projectID"))
+	if err != nil {
+		testsuit.T.Fail(fmt.Errorf("failed to convert project ID to integer: %v", err))
+	}
+	mrID, err := strconv.Atoi(store.GetScenarioData("mrID"))
+	if err != nil {
+		testsuit.T.Fail(fmt.Errorf("failed to convert MR ID to integer: %v", err))
+	}
+
+	sinceStr := store.GetScenarioData("aiCommentSince")
+	sinceTime, err := time.Parse(time.RFC3339Nano, sinceStr)
+	if err != nil {
+		testsuit.T.Fail(fmt.Errorf("invalid aiCommentSince timestamp %q: %v", sinceStr, err))
+	}
+
+	const marker = "pac_ai_gemini_failure_analysis"
+	const maxAttempts = 30
+	const delay = 10 * time.Second
+
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		notes, _, err := client.Notes.ListMergeRequestNotes(projectID, mrID, &gitlab.ListMergeRequestNotesOptions{})
+		if err != nil {
+			testsuit.T.Fail(fmt.Errorf("failed to list MR notes for MR %d in project %d: %v", mrID, projectID, err))
+		}
+		for _, n := range notes {
+			if !strings.Contains(strings.ToLower(n.Body), marker) {
+				continue
+			}
+			updated := n.UpdatedAt
+			created := n.CreatedAt
+			after := false
+			if updated != nil && updated.After(sinceTime) {
+				after = true
+			} else if created != nil && created.After(sinceTime) {
+				after = true
+			}
+			if !after {
+				continue
+			}
+
+			log.Printf("Found AI analysis comment on MR %d:\n", mrID)
+			return
+		}
+
+		log.Printf("AI analysis comment not found yet on MR %d (attempt %d/%d); sleeping %s...", mrID, attempt, maxAttempts, delay)
+		time.Sleep(delay)
+	}
+
+	testsuit.T.Fail(fmt.Errorf("AI analysis comment not found on MR %d after %d attempts", mrID, maxAttempts))
+}

specs/pac/pac-gitlab.spec

@@ -62,3 +62,24 @@ Steps:
   * "2" pipelinerun(s) should be present within "10" seconds
   * Validate "pull_request" PipelineRun for "success"
   * Cleanup PAC
+
+## Configure PAC AI analysis in GitLab Project: PIPELINES-30-TC04
+Tags: pac, e2e, ai-analysis
+Component: PAC
+Level: Integration
+Type: Functional
+Importance: High
+
+This scenario tests AI-powered failure analysis on a failed pull_request PipelineRun in GitLab.
+
+Steps:
+  * Validate PAC Info Install
+  * Setup Gitlab Client
+  * Create Smee deployment
+  * Configure AI analysis for PAC Repository
+  * Configure GitLab repo for "pull_request" in "main"
+  * Make pull_request PipelineRun fail
+  * Configure PipelineRun
+  * Validate "pull_request" PipelineRun for "fail"
+  * Validate AI summary comment is added in MR
+  * Cleanup PAC

steps/pac/pac.go

@@ -34,11 +34,6 @@ var _ = gauge.Step("Trigger push event on main branch", func() {
 	pac.TriggerPushOnForkMain()
 })
 
-var _ = gauge.Step("Validate PipelineRun for <state>", func(state string) {
-	pipelineName := pac.GetPipelineNameFromMR()
-	pipelines.ValidatePipelineRun(store.Clients(), pipelineName, state, store.Namespace())
-})
-
 var _ = gauge.Step("Validate <event_type> PipelineRun for <state>", func(event_type, state string) {
 	switch event_type {
 	case "pull_request":
@@ -60,6 +55,18 @@ var _ = gauge.Step("Update Annotation <annotationKey> with <annotationValue>", f
 	pac.UpdateAnnotation(annotationKey, annotationValue)
 })
 
+var _ = gauge.Step("Configure AI analysis for PAC Repository", func() {
+	pac.ConfigureRepositoryAISettings()
+})
+
+var _ = gauge.Step("Make pull_request PipelineRun fail", func() {
+	pac.MakePullRequestPipelineFail()
+})
+
+var _ = gauge.Step("Validate AI summary comment is added in MR", func() {
+	pac.ValidateAIMRComment()
+})
+
 var _ = gauge.Step("Add Comment <comment> in MR", func(comment string) {
 	pac.AddComment(comment)
 })

testdata/pac/pull-request-go-fail.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  annotations:
+    pipelinesascode.tekton.dev/on-target-branch: "[main]"
+    pipelinesascode.tekton.dev/on-event: "[pull_request]"
+  name: go
+spec:
+  pipelineSpec:
+    tasks:
+      - name: error-me
+        taskSpec:
+          steps:
+            - name: eror-me-robot
+              image: registry.access.redhat.com/ubi8/ubi-micro:8.4
+              script: | 
+               cat <<EOF
+                === RUN   TestCreateBasicAuthSecret/different_git_user
+                basic_auth_test.go:101: assertion failed: https://superman:supersecrete@forge/bat/man (secret.StringData[".git-credentials"] string) != https://superman:supersecrete@forge/super/man (tt.expectedGitCredentials string)
+                --- FAIL: TestCreateBasicAuthSecret (0.00s)
+                    --- PASS: TestCreateBasicAuthSecret/Target_secret_not_there (0.00s)
+                    --- PASS: TestCreateBasicAuthSecret/Use_clone_URL (0.00s)
+                    --- PASS: TestCreateBasicAuthSecret/Target_secret_already_there (0.00s)
+                    --- PASS: TestCreateBasicAuthSecret/Lowercase_secrets (0.00s)
+                    --- PASS: TestCreateBasicAuthSecret/Use_clone_URL#01 (0.00s)
+                    --- FAIL: TestCreateBasicAuthSecret/different_git_user (0.00s)
+                FAIL
+                FAIL	github.com/openshift-pipelines/pipelines-as-code/pkg/secrets	0.030s
+                FAIL
+               EOF
+               exit 1