diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index f236dc2e8b3..fe00b76aa63 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -18,6 +18,7 @@ require ( github.com/mark3labs/mcp-go v0.41.1 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 @@ -76,7 +77,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index 75ff859d9e1..258ff4e9440 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -938,9 +938,14 @@ func (a *InitAction) downloadAgentYaml( return nil, "", fmt.Errorf("marshaling agent manifest to YAML: %w", err) } content = manifestBytes + } else { + return nil, "", fmt.Errorf("unrecognized manifest pointer format: %s. Expected local file path, GitHub URL, or registry URL", manifestPointer) } // Parse and validate the YAML content against AgentManifest structure + if len(content) == 0 { + return nil, "", fmt.Errorf("manifest content is empty or could not be retrieved") + } agentManifest, err := agent_yaml.LoadAndValidateAgentManifest(content) if err != nil { return nil, "", fmt.Errorf("AgentManifest %w", err) @@ -968,11 +973,19 @@ func (a *InitAction) downloadAgentYaml( } } - agentId := agentManifest.Name + var agentName string + + if containerTemplate, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok { + agentName = containerTemplate.Name + } else if promptTemplate, ok := agentManifest.Template.(agent_yaml.PromptAgent); ok { + agentName = promptTemplate.Name + } else { + return nil, "", fmt.Errorf("unsupported agent template type") + } - // Use targetDir if provided or set to local file pointer, otherwise default to "src/{agentId}" + // Use targetDir if provided or set to local file pointer, otherwise default to "src/{agentName}" if targetDir == "" { - targetDir = filepath.Join("src", agentId) + targetDir = filepath.Join("src", agentName) } // Create target directory if it doesn't exist diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go new file mode 100644 index 00000000000..87111c380b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package deployTests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" + + "azureaiagent/test/integrationTests/testUtilities" + + "github.com/stretchr/testify/require" +) + +const testManifestURL = "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml" + +// Shared test suite instance for deploy tests +var deployTestSuite *testUtilities.IntegrationTestSuite + +func TestMain(m *testing.M) { + // Initialize logging configuration + testUtilities.InitializeLogging() + + testUtilities.SetCurrentTestName("SETUP") + testUtilities.Logf("Starting deploy test suite") + + // Setup test suite once for all deploy tests + suite, err := testUtilities.SetupTestSuite() + if err != nil { + testUtilities.Logf("Failed to setup test suite: %v", err) + os.Exit(1) + } + deployTestSuite = suite + + // Run tests + code := m.Run() + + // Cleanup + testUtilities.SetCurrentTestName("CLEANUP") + testUtilities.Logf("Running cleanup") + if suite.CleanupFunc != nil { + suite.CleanupFunc() + } + testUtilities.Logf("Deploy test suite completed") + + os.Exit(code) +} + +func TestDeployCommand_Integration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Ensure test suite is initialized + require.NotNil(t, deployTestSuite, "Deploy test suite should be initialized") + testUtilities.SetCurrentTestName("DEPLOY") + testUtilities.Logf("Running integration tests with project ID: %s", deployTestSuite.ProjectID) + + tests := []struct { + name string + agentName string + manifestURL string + wantErr bool + }{ + { + name: "DeployWithValidManifest", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testUtilities.SetCurrentTestName(tt.name) + testUtilities.Logf("Running test: %s", tt.name) + + // Execute init command + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, "", deployTestSuite) + + require.NoError(t, err) + + // Verify expected files were created + testUtilities.VerifyInitializedProject(t, deployTestSuite, "", tt.agentName) + + // Execute deploy command + agentVersion, err := testUtilities.ExecuteDeployCommandForAgent(context.Background(), tt.agentName, deployTestSuite) + if tt.wantErr { + require.Error(t, err) + testUtilities.Logf("Test completed (expected error)") + return + } + + require.NoError(t, err) + if agentVersion != "" { + testUtilities.Logf("Agent deployed with version: %s", agentVersion) + } + + // Wait for agent service to be fully ready after deployment + testUtilities.Logf("Waiting 30 seconds for agent service to initialize...") + time.Sleep(30 * time.Second) + + // Verify deployment was successful + verifyAgentDeployment(t, tt.agentName, agentVersion) + testUtilities.Logf("Test completed successfully") + }) + } +} + +// verifyAgentDeployment checks that the agent was deployed successfully by making API calls +func verifyAgentDeployment(t *testing.T, agentName string, agentVersion string) { + t.Helper() + testUtilities.Logf("Verifying deployment for %s (version: %s)...", agentName, agentVersion) + + // Get required environment variables from azd environment + endpoint, err := deployTestSuite.GetAzdEnvValue("AZURE_AI_PROJECT_ENDPOINT") + require.NoError(t, err, "Failed to get AZURE_AI_PROJECT_ENDPOINT") + require.NotEmpty(t, endpoint, "AZURE_AI_PROJECT_ENDPOINT should be set") + testUtilities.Logf("Using endpoint: %s", endpoint) + + apiVersion := getEnvOrDefault("AGENT_API_VERSION", "2025-05-15-preview") + // Agent version is required - fail if not provided + require.NotEmpty(t, agentVersion, "Agent version should be parsed from deploy command output") + testMessage := getEnvOrDefault("AGENT_TEST_MESSAGE", "What is 2 + 2?") + + // Get Azure access token + token, err := getAzureAccessToken(t) + require.NoError(t, err, "Failed to get Azure access token") + testUtilities.Logf("Successfully obtained Azure access token") + + // Step 1: Create a conversation + conversationID, err := createConversation(t, endpoint, apiVersion, token) + require.NoError(t, err, "Failed to create conversation") + testUtilities.Logf("Created conversation with ID: %s", conversationID) + + // Step 2: Get response from agent + err = testAgentResponse(t, endpoint, apiVersion, token, agentName, agentVersion, testMessage) + require.NoError(t, err, "Failed to get valid response from agent") + + testUtilities.Logf("Deployment verification completed successfully") +} + +// getEnvOrDefault gets an environment variable or returns a default value +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// getAzureAccessToken obtains an Azure access token using az cli +func getAzureAccessToken(t *testing.T) (string, error) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "az", "account", "get-access-token", "--resource", "https://ai.azure.com", "--query", "accessToken", "-o", "tsv") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get access token: %w", err) + } + + token := strings.TrimSpace(string(output)) + if token == "" { + return "", fmt.Errorf("access token is empty") + } + + return token, nil +} + +// createConversation creates a new conversation and returns its ID +func createConversation(t *testing.T, endpoint, apiVersion, token string) (string, error) { + t.Helper() + + conversationURL := fmt.Sprintf("%s/openai/conversations?api-version=%s", endpoint, apiVersion) + + payload := map[string]interface{}{ + "metadata": map[string]string{ + "test_session": "integration_test_agent_response", + }, + } + + payloadBytes, err := json.Marshal(payload) + require.NoError(t, err, "Failed to marshal conversation payload") + + req, err := http.NewRequest("POST", conversationURL, bytes.NewBuffer(payloadBytes)) + require.NoError(t, err, "Failed to create conversation request") + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: 2 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("conversation request failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to create conversation (status %d): %s", resp.StatusCode, string(body)) + } + + var conversationData map[string]interface{} + if err := json.Unmarshal(body, &conversationData); err != nil { + return "", fmt.Errorf("failed to parse conversation response: %w", err) + } + + conversationID, ok := conversationData["id"].(string) + if !ok || conversationID == "" { + return "", fmt.Errorf("conversation ID not found in response") + } + + return conversationID, nil +} + +// testAgentResponse sends a test message to the agent and verifies the response +func testAgentResponse(t *testing.T, endpoint, apiVersion, token, agentName, agentVersion, testMessage string) error { + t.Helper() + + requestURL := fmt.Sprintf("%s/openai/responses?api-version=%s", endpoint, apiVersion) + + payload := map[string]interface{}{ + "agent": map[string]string{ + "type": "agent_reference", + "name": agentName, + "version": agentVersion, + }, + "input": testMessage, + } + + payloadBytes, err := json.Marshal(payload) + require.NoError(t, err, "Failed to marshal agent request payload") + + testUtilities.Logf("Agent request payload: %s", string(payloadBytes)) + + req, err := http.NewRequest("POST", requestURL, bytes.NewBuffer(payloadBytes)) + require.NoError(t, err, "Failed to create agent request") + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Set("Content-Type", "application/json") + + // Increase timeout for agent response - agents can take time to process + client := &http.Client{Timeout: 2 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("agent request failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get response from agent (status %d): %s", resp.StatusCode, string(body)) + } + + var responseData map[string]interface{} + if err := json.Unmarshal(body, &responseData); err != nil { + return fmt.Errorf("failed to parse agent response: %w", err) + } + + testUtilities.Logf("Agent response data: %s", string(body)) + + // Verify response doesn't contain errors + if errorData, hasError := responseData["error"]; hasError && errorData != nil { + return fmt.Errorf("agent response contains error: %v", errorData) + } + + // Verify response has output + output, hasOutput := responseData["output"] + if !hasOutput { + return fmt.Errorf("response missing 'output' field") + } + + // Check if output is a string or array and verify it's not empty + switch v := output.(type) { + case string: + if len(v) == 0 { + return fmt.Errorf("response output string is empty") + } + testUtilities.Logf("Agent response output (string): %s", v) + case []interface{}: + if len(v) == 0 { + return fmt.Errorf("response output array is empty") + } + testUtilities.Logf("Agent response output (array with %d items)", len(v)) + default: + testUtilities.Logf("Agent response output type: %T", v) + } + + testUtilities.Logf("Agent response validation successful") + return nil +} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go new file mode 100644 index 00000000000..bc1bf9bbe11 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package initTests + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "azureaiagent/test/integrationTests/testUtilities" + + "github.com/stretchr/testify/require" +) + +const testManifestURL = "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml" + +// Shared test suite instance for init tests +var testSuite *testUtilities.IntegrationTestSuite + +// TestMain provides package-level setup and teardown for integration tests +func TestMain(m *testing.M) { + // Initialize logging configuration + testUtilities.InitializeLogging() + + testUtilities.SetCurrentTestName("SETUP") + testUtilities.Logf("Starting integration test suite") + + // Setup + suite, err := testUtilities.SetupTestSuite() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to setup integration test suite: %v\n", err) + os.Exit(1) + } + testSuite = suite + + // Run tests + code := m.Run() + + // Cleanup + testUtilities.SetCurrentTestName("CLEANUP") + testUtilities.Logf("Running cleanup") + if testSuite != nil && testSuite.CleanupFunc != nil { + testSuite.CleanupFunc() + } + testUtilities.Logf("Integration test suite completed") + + os.Exit(code) +} + +func TestInitCommand_Integration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Ensure test suite is initialized + require.NotNil(t, testSuite, "Test suite should be initialized") + testUtilities.SetCurrentTestName("INIT") + testUtilities.Logf("Running integration tests with project ID: %s", testSuite.ProjectID) + + // Create temporary directory for separated src dir test + tempDir := t.TempDir() + + tests := []struct { + name string + agentName string + manifestURL string + targetDir string + wantErr bool + }{ + { + name: "InitWithValidManifestDefaultSrc", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + targetDir: "", + wantErr: false, + }, + { + name: "InitWithValidManifestRelativeSrc", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + targetDir: filepath.Join("src", "calculator-agent-test"), + wantErr: false, + }, + { + name: "InitWithValidManifestExternalSrc", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + targetDir: filepath.Join(tempDir, "calculator-agent-test"), + wantErr: false, + }, + { + name: "InitWithInvalidManifest", + manifestURL: "https://invalid-url.com/agent.yaml", + targetDir: "invalid-agent-test", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testUtilities.SetCurrentTestName(tt.name) + testUtilities.Logf("Running test: %s", tt.name) + + // Execute init command + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, tt.targetDir, testSuite) + if tt.wantErr { + require.Error(t, err) + testUtilities.Logf("Test completed (expected error)") + return + } + + require.NoError(t, err) + + // Verify expected files were created + testUtilities.VerifyInitializedProject(t, testSuite, tt.targetDir, tt.agentName) + testUtilities.Logf("Test completed successfully") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go new file mode 100644 index 00000000000..8e95be0d19a --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go @@ -0,0 +1,292 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package testUtilities + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" +) + +// IntegrationTestSuite holds shared test resources and state +type IntegrationTestSuite struct { + AzdBinary string + AzdProjectDir string + CleanupFunc func() + ProjectID string // Retrieved from azd env after provisioning +} + +// SetupTestSuite initializes shared test resources +func SetupTestSuite() (*IntegrationTestSuite, error) { + Logf("Setting up test suite") + suite := &IntegrationTestSuite{} + + // Find azd binary + azdPath, err := findAzdBinary() + if err != nil { + return nil, fmt.Errorf("failed to find azd binary: %w", err) + } + suite.AzdBinary = azdPath + Logf("Found azd binary: %s", azdPath) + + // Verify azd binary works + if err := suite.verifyAzdBinary(); err != nil { + return nil, fmt.Errorf("azd binary verification failed: %w", err) + } + + // Create test environment directory + azdProjectDir, err := os.MkdirTemp("", "azd-ai-agents-test-env-*") + if err != nil { + return nil, fmt.Errorf("failed to create test environment directory: %w", err) + } + suite.AzdProjectDir = azdProjectDir + Logf("Created test environment: %s", azdProjectDir) + suite.CleanupFunc = func() { + // Run azd down to clean up Azure resources + suite.runAzdDown() + // Remove local test directory + if err := os.RemoveAll(azdProjectDir); err != nil { + Logf("Warning: failed to remove test directory: %v", err) + } + } + + // Initialize test environment with azd template + Logf("Initializing Azure environment...") + if err := suite.initializeAzdProject(); err != nil { + suite.CleanupFunc() + return nil, fmt.Errorf("failed to initialize test environment: %w", err) + } + Logf("Azure environment ready (Project ID: %s)", suite.ProjectID) + + return suite, nil +} + +// verifyAzdBinary ensures the azd binary is working and has the ai agent extension +func (s *IntegrationTestSuite) verifyAzdBinary() error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Test basic azd functionality + cmd := exec.CommandContext(ctx, s.AzdBinary, "version") + var outputBuilder strings.Builder + cmd.Stdout = &outputBuilder + cmd.Stderr = &outputBuilder + err := cmd.Run() + output := outputBuilder.String() + LogCommandOutput("azd version", []byte(output)) + if err != nil { + return fmt.Errorf("azd version command failed: %w", err) + } + + // Test ai agent extension is available + cmd = exec.CommandContext(ctx, s.AzdBinary, "ai", "agent", "--help") + outputBuilder.Reset() + cmd.Stdout = &outputBuilder + cmd.Stderr = &outputBuilder + err = cmd.Run() + output = outputBuilder.String() + LogCommandOutput("azd ai agent --help", []byte(output)) + if err != nil { + return fmt.Errorf("azd ai agent extension not available: %w", err) + } + + return nil +} + +// initializeAzdProject sets up the test environment with azd template and provisions resources +func (s *IntegrationTestSuite) initializeAzdProject() error { + // Change to test environment directory once + originalDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + + if err := os.Chdir(s.AzdProjectDir); err != nil { + return fmt.Errorf("failed to change to test environment directory: %w", err) + } + defer os.Chdir(originalDir) + + // Run azd init command + if err := s.runAzdInit(true); err != nil { + return fmt.Errorf("failed to run azd init: %w", err) + } + + // Verify the environment was created successfully + if err := s.verifyTestEnvironment(); err != nil { + return fmt.Errorf("test environment verification failed: %w", err) + } + + // Set required environment variable for provisioning an ACR + if err := s.SetAzdEnvValue("ENABLE_HOSTED_AGENTS", "true"); err != nil { + return fmt.Errorf("failed to set ENABLE_HOSTED_AGENTS: %w", err) + } + + // Initialize a calculator agent into the project so we have the model we need + if err := ExecuteInitCommandForAgent(context.Background(), + "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml", + "", s); err != nil { + return fmt.Errorf("failed to initialize calculator agent: %w", err) + } + + // Run azd up to provision Azure resources + if err := s.runAzdUp(); err != nil { + return fmt.Errorf("failed to run azd up: %w", err) + } + + // Retrieve project ID from azd environment + projectID, err := s.GetAzdEnvValue("AZURE_AI_PROJECT_ID") + if err != nil { + return fmt.Errorf("failed to retrieve project ID: %w", err) + } + if projectID == "" { + return fmt.Errorf("AZURE_AI_PROJECT_ID is empty") + } + s.ProjectID = projectID + + return nil +} + +// runAzdInit executes the azd init command +func (s *IntegrationTestSuite) runAzdInit(withTemplate bool) error { + Logf("Running azd init...") + + // Generate a unique environment name using a short UUID suffix + suffix := uuid.New().String()[:8] + + args := []string{ + "init", + "-e", fmt.Sprintf("azd-extension-integration-tests-%s", suffix), + "--location", "northcentralus", + "--subscription", "827cb315-a120-4b3d-bd80-93f7b3126af2", + "--no-prompt", + } + + if withTemplate { + args = append(args, "-t", "Azure-Samples/azd-ai-starter-basic") + } + + _, err := executeAzdCommand(context.Background(), s, 5*time.Minute, args) + if err != nil { + return fmt.Errorf("azd init command failed: %w", err) + } + + return nil +} + +// runAzdUp executes the azd up command +func (s *IntegrationTestSuite) runAzdUp() error { + Logf("Running azd up (provisioning Azure resources)...") + + args := []string{"up", "--no-prompt"} + + _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) + if err != nil { + return fmt.Errorf("azd up command failed: %w", err) + } + + return nil +} + +// runAzdDown executes the azd down command to clean up Azure resources +func (s *IntegrationTestSuite) runAzdDown() { + Logf("Cleaning up Azure resources...") + + args := []string{"down", "--force", "--purge", "--no-prompt"} + + _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) + if err != nil { + Logf("Warning: Azure cleanup failed: %v", err) + } +} + +// GetAzdEnvValue retrieves an environment variable value from the azd environment +func (s *IntegrationTestSuite) GetAzdEnvValue(key string) (string, error) { + Logf("Running azd env get-value %s...", key) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + args := []string{"env", "get-value", key} + + cmd := exec.CommandContext(ctx, s.AzdBinary, args...) + cmd.Dir = s.AzdProjectDir + + // Capture output to get the value + output, err := cmd.Output() + LogCommandOutput(fmt.Sprintf("azd env get-value %s", key), output) + if err != nil { + return "", fmt.Errorf("failed to get %s from azd env: %w", key, err) + } + + // Trim whitespace and return the value + return strings.TrimSpace(string(output)), nil +} + +// SetAzdEnvValue sets an environment variable value in the azd environment +func (s *IntegrationTestSuite) SetAzdEnvValue(key string, value string) error { + Logf("Running azd env set %s %s...", key, value) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + args := []string{"env", "set", key, value} + + cmd := exec.CommandContext(ctx, s.AzdBinary, args...) + cmd.Dir = s.AzdProjectDir + + // Capture output to get the value + output, err := cmd.Output() + LogCommandOutput(fmt.Sprintf("azd env set %s", key), output) + if err != nil { + return fmt.Errorf("failed to set %s in azd env: %w", key, err) + } + + return nil +} + +// verifyTestEnvironment ensures the test environment was set up correctly +func (s *IntegrationTestSuite) verifyTestEnvironment() error { + // Check for expected files/directories in the test environment + expectedFiles := []string{ + filepath.Join(s.AzdProjectDir, "azure.yaml"), + filepath.Join(s.AzdProjectDir, ".azure"), + } + + for _, path := range expectedFiles { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("expected path %s not found: %w", path, err) + } + } + + return nil +} + +// findAzdBinary locates the azd binary for testing +func findAzdBinary() (string, error) { + // First try to find azd in PATH (the main CLI with extension support) + azdPath, err := exec.LookPath("azd") + if err == nil { + return azdPath, nil + } + + // Fallback to local binary for development scenarios + extensionRoot := filepath.Join("..", "..", "..") + localBinary := filepath.Join(extensionRoot, "azureaiagent") + if os.PathSeparator == '\\' { + localBinary += ".exe" + } + + if _, err := os.Stat(localBinary); err == nil { + return localBinary, nil + } + + return "", &InitError{ + Message: "azd binary not found in PATH and local binary not available", + Err: err, + } +} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go new file mode 100644 index 00000000000..07e45777d68 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package testUtilities + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + // Verbose logging flag + verboseLogging bool + // Current test context for logging + currentTestName string +) + +// InitializeLogging sets up logging configuration for integration tests +func InitializeLogging() { + // Check if verbose logging is enabled by looking at command line args + for _, arg := range os.Args { + if arg == "-v" || arg == "-test.v" || arg == "-test.v=true" { + verboseLogging = true + break + } + } + if verboseLogging { + log.SetFlags(log.LstdFlags | log.Lshortfile) + } +} + +// Logf logs a message if verbose logging is enabled +func Logf(format string, args ...interface{}) { + if verboseLogging { + prefix := "[INTEGRATION]" + if currentTestName != "" { + prefix = fmt.Sprintf("[%s]", currentTestName) + } + log.Printf(prefix+" "+format, args...) + } +} + +// LogCommandOutput logs command output if verbose logging is enabled +func LogCommandOutput(cmd string, output []byte) { + if verboseLogging && len(output) > 0 { + log.Printf("[COMMAND] %s output:\n%s", cmd, string(output)) + } +} + +// SetCurrentTestName sets the current test name for logging context +func SetCurrentTestName(name string) { + currentTestName = name +} + +// ExecuteInitCommandForAgent executes the AI agent init command with the given parameters +func ExecuteInitCommandForAgent(ctx context.Context, manifestURL, targetPath string, testSuite *IntegrationTestSuite) error { + // Prepare command arguments + args := []string{"ai", "agent", "init", "--no-prompt"} + + if manifestURL != "" { + args = append(args, "--manifest", manifestURL) + } + + if targetPath != "" { + args = append(args, "--src", targetPath) + } + + // Add project-id retrieved from azd environment + if testSuite.ProjectID != "" { + args = append(args, "--project-id", testSuite.ProjectID) + } + + _, err := executeAzdCommand(ctx, testSuite, 2*time.Minute, args) + if err != nil { + return err + } + + // Debug: Show files in working directory as well + if files, err := os.ReadDir(testSuite.AzdProjectDir); err == nil { + Logf("Files in working directory (%s):", testSuite.AzdProjectDir) + for _, file := range files { + Logf(" - %s (dir: %v)", file.Name(), file.IsDir()) + } + } else { + Logf("Could not read working directory: %v", err) + } + + return nil +} + +// VerifyInitializedProject verifies that the expected files and structure were created +func VerifyInitializedProject(t *testing.T, testSuite *IntegrationTestSuite, srcDir string, agentName string) { + t.Helper() + if srcDir == "" { + srcDir = filepath.Join(testSuite.AzdProjectDir, "src", agentName) + } else if !filepath.IsAbs(srcDir) { + // If srcDir is relative, make it relative to the azd project directory + srcDir = filepath.Join(testSuite.AzdProjectDir, srcDir) + } + Logf("Verifying project at path: %s", srcDir) + + // Verify basic project structure exists + require.DirExists(t, srcDir, "Project directory should exist") + + // List all files in the project directory for debugging + if files, err := os.ReadDir(srcDir); err == nil { + Logf("Files in target directory:") + for _, file := range files { + Logf(" - %s (dir: %v)", file.Name(), file.IsDir()) + } + } else { + Logf("Could not read target directory: %v", err) + } + + // Verify expected files exist in target directory + expectedFilesInTarget := []string{ + "agent.yaml", + "Dockerfile", + "main.py", + "requirements.txt", + } + + for _, file := range expectedFilesInTarget { + fullPath := filepath.Join(srcDir, file) + Logf("Checking for file: %s", fullPath) + require.FileExists(t, fullPath, "Expected file %s should exist in target directory", file) + + // Verify file is not empty + info, err := os.Stat(fullPath) + require.NoError(t, err) + require.Greater(t, info.Size(), int64(0), "File %s should not be empty", file) + } + + // Verify azure.yaml was updated in the test environment + azureYamlPath := filepath.Join(testSuite.AzdProjectDir, "azure.yaml") + Logf("Checking for azure.yaml in test environment: %s", azureYamlPath) + require.FileExists(t, azureYamlPath, "azure.yaml should exist in test environment directory") + + // Additional verifications can be added here based on your init command's behavior + // For example, checking for infra/ directory, src/ directory, etc. +} + +// InitError represents an error from the init command +type InitError struct { + Message string + Err error +} + +func (e *InitError) Error() string { + if e.Err != nil { + return e.Message + ": " + e.Err.Error() + } + return e.Message +} + +func (e *InitError) Unwrap() error { + return e.Err +} + +// executeAzdCommand executes an azd command and returns output and agent version if available +func executeAzdCommand(ctx context.Context, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + Logf("Executing command: %s %s", testSuite.AzdBinary, strings.Join(args, " ")) + Logf("Working directory: %s", testSuite.AzdProjectDir) + + // Execute the command + cmd := exec.CommandContext(ctx, testSuite.AzdBinary, args...) + cmd.Dir = testSuite.AzdProjectDir + + // Use a strings.Builder to capture output without truncation + var outputBuilder strings.Builder + cmd.Stdout = &outputBuilder + cmd.Stderr = &outputBuilder + + // Run the command + err := cmd.Run() + output := outputBuilder.String() + + // Log the full output + LogCommandOutput(strings.Join(args, " "), []byte(output)) + + if err != nil { + Logf("Command failed with error: %v", err) + return "", &InitError{ + Message: output, + Err: err, + } + } + Logf("Command completed successfully") + + return output, nil +} + +func parseOutputForAgentVersion(output string) string { + // Parse the agent version from the output + // Look for pattern: "Agent endpoint: .../agents/{agentName}/versions/{version}" + versionRegex := regexp.MustCompile(`Agent endpoint:.*?/agents/[^/]+/versions/(\d+)`) + matches := versionRegex.FindStringSubmatch(output) + + var agentVersion string + if len(matches) > 1 { + agentVersion = matches[1] + Logf("Parsed agent version: %s", agentVersion) + } else { + // Don't fail in the case we can't parse the agent version, just log a warning + // This allows tests to individually handle whether they require the version or not + Logf("Warning: Could not parse agent version from output") + } + + return agentVersion +} + +// ExecuteUpCommandForAgent executes the AZD up command with the given parameters +// Returns the deployed agent version number if successful +func ExecuteUpCommandForAgent(ctx context.Context, testSuite *IntegrationTestSuite) (string, error) { + args := []string{"up", "--no-prompt"} + + output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) + if err != nil { + return "", err + } + + agentVersion := parseOutputForAgentVersion(output) + + return agentVersion, nil +} + +// ExecuteDeployCommandForAgent executes the AZD deploy command with the given parameters +// Returns the deployed agent version number if successful +func ExecuteDeployCommandForAgent(ctx context.Context, agentName string, testSuite *IntegrationTestSuite) (string, error) { + // Prepare command arguments + args := []string{"deploy"} + if agentName != "" { + args = append(args, agentName) + } + args = append(args, "--no-prompt") + + output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) + if err != nil { + return "", err + } + + agentVersion := parseOutputForAgentVersion(output) + + return agentVersion, nil +}