From 04c6cf8bee72a1738369b84ec03c9cf0988f53b8 Mon Sep 17 00:00:00 2001 From: trangevi Date: Fri, 21 Nov 2025 09:14:43 -0800 Subject: [PATCH 1/7] Initial test setup Signed-off-by: trangevi --- .../azure.ai.agents/internal/cmd/init.go | 5 + .../test/integrationTests/init_test.go | 510 ++++++++++++++++++ 2 files changed, 515 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index ac1812ca16c..24afae20330 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -936,9 +936,14 @@ func (a *InitAction) downloadAgentYaml( return nil, "", fmt.Errorf("marshaling agent manifest to YAML: %w", err) } content = manifestBytes + } else { + return nil, "", fmt.Errorf("unrecognized manifest pointer format: %s. Expected local file path, GitHub URL, or registry URL", manifestPointer) } // Parse and validate the YAML content against AgentManifest structure + if len(content) == 0 { + return nil, "", fmt.Errorf("manifest content is empty or could not be retrieved") + } agentManifest, err := agent_yaml.LoadAndValidateAgentManifest(content) if err != nil { return nil, "", fmt.Errorf("AgentManifest %w", err) diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go new file mode 100644 index 00000000000..c1f2b21c5bf --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go @@ -0,0 +1,510 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package integrationTests + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const testManifestURL = "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/microsoft/python/getting-started-agents/hosted-agents/calculator-agent/agent.yaml" + +var ( + // Global test state + testSuite *IntegrationTestSuite + // Verbose logging flag + verboseLogging bool + // Current test context for logging + currentTestName string +) + +// logf logs a message if verbose logging is enabled +func logf(format string, args ...interface{}) { + if verboseLogging { + prefix := "[INTEGRATION]" + if currentTestName != "" { + prefix = fmt.Sprintf("[%s]", currentTestName) + } + log.Printf(prefix+" "+format, args...) + } +} + +// logCommandOutput logs command output if verbose logging is enabled +func logCommandOutput(cmd string, output []byte) { + if verboseLogging && len(output) > 0 { + log.Printf("[COMMAND] %s output:\n%s", cmd, string(output)) + } +} + +// IntegrationTestSuite holds shared test resources and state +type IntegrationTestSuite struct { + azdBinary string + testEnvDir string + cleanupFunc func() + projectID string // Retrieved from azd env after provisioning +} + +// TestMain provides package-level setup and teardown for integration tests +func TestMain(m *testing.M) { + // Check if verbose logging is enabled by looking at command line args + for _, arg := range os.Args { + if arg == "-v" || arg == "-test.v" || arg == "-test.v=true" { + verboseLogging = true + break + } + } + if verboseLogging { + log.SetFlags(log.LstdFlags | log.Lshortfile) + } + + currentTestName = "SETUP" + logf("Starting integration test suite") + + // Setup + suite, err := setupIntegrationTestSuite() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to setup integration test suite: %v\n", err) + os.Exit(1) + } + testSuite = suite + + // Run tests + code := m.Run() + + // Cleanup + currentTestName = "CLEANUP" + logf("Running cleanup") + if testSuite != nil && testSuite.cleanupFunc != nil { + testSuite.cleanupFunc() + } + logf("Integration test suite completed") + + os.Exit(code) +} + +// setupIntegrationTestSuite initializes shared test resources +func setupIntegrationTestSuite() (*IntegrationTestSuite, error) { + logf("Setting up integration test suite") + suite := &IntegrationTestSuite{} + + // Find azd binary + azdPath, err := findAzdBinary() + if err != nil { + return nil, fmt.Errorf("failed to find azd binary: %w", err) + } + suite.azdBinary = azdPath + logf("Found azd binary: %s", azdPath) + + // Verify azd binary works + if err := suite.verifyAzdBinary(); err != nil { + return nil, fmt.Errorf("azd binary verification failed: %w", err) + } + + // Create test environment directory + testEnvDir, err := os.MkdirTemp("", "azd-ai-agents-test-env-*") + if err != nil { + return nil, fmt.Errorf("failed to create test environment directory: %w", err) + } + suite.testEnvDir = testEnvDir + logf("Created test environment: %s", testEnvDir) + suite.cleanupFunc = func() { + // Run azd down to clean up Azure resources + suite.runAzdDown() + // Remove local test directory + if err := os.RemoveAll(testEnvDir); err != nil { + logf("Warning: failed to remove test directory: %v", err) + } + } + + // Initialize test environment with azd template + logf("Initializing Azure environment...") + if err := suite.initializeTestEnvironment(); err != nil { + suite.cleanupFunc() + return nil, fmt.Errorf("failed to initialize test environment: %w", err) + } + logf("Azure environment ready (Project ID: %s)", suite.projectID) + + return suite, nil +} + +// verifyAzdBinary ensures the azd binary is working and has the ai agent extension +func (s *IntegrationTestSuite) verifyAzdBinary() error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Test basic azd functionality + cmd := exec.CommandContext(ctx, s.azdBinary, "version") + output, err := cmd.CombinedOutput() + logCommandOutput("azd version", output) + if err != nil { + return fmt.Errorf("azd version command failed: %w", err) + } + + // Test ai agent extension is available + cmd = exec.CommandContext(ctx, s.azdBinary, "ai", "agent", "--help") + output, err = cmd.CombinedOutput() + logCommandOutput("azd ai agent --help", output) + if err != nil { + return fmt.Errorf("azd ai agent extension not available: %w", err) + } + + return nil +} + +// initializeTestEnvironment sets up the test environment with azd template and provisions resources +func (s *IntegrationTestSuite) initializeTestEnvironment() error { + // Change to test environment directory once + originalDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + + if err := os.Chdir(s.testEnvDir); err != nil { + return fmt.Errorf("failed to change to test environment directory: %w", err) + } + defer os.Chdir(originalDir) + + // Run azd init command + if err := s.runAzdInit(); err != nil { + return fmt.Errorf("failed to run azd init: %w", err) + } + + // Verify the environment was created successfully + if err := s.verifyTestEnvironment(); err != nil { + return fmt.Errorf("test environment verification failed: %w", err) + } + + // Run azd up to provision Azure resources + if err := s.runAzdUp(); err != nil { + return fmt.Errorf("failed to run azd up: %w", err) + } + + // Retrieve project ID from azd environment + if err := s.retrieveProjectID(); err != nil { + return fmt.Errorf("failed to retrieve project ID: %w", err) + } + + return nil +} + +// runAzdInit executes the azd init command +func (s *IntegrationTestSuite) runAzdInit() error { + logf("Running azd init...") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + args := []string{ + "init", + "-t", "Azure-Samples/azd-ai-starter-basic", + "-e", "trangevi-test", + "--location", "westus2", + "--subscription", "827cb315-a120-4b3d-bd80-93f7b3126af2", + "--no-prompt", + } + + cmd := exec.CommandContext(ctx, s.azdBinary, args...) + cmd.Dir = s.testEnvDir + + // Capture output for debugging + output, err := cmd.CombinedOutput() + logCommandOutput("azd init", output) + if err != nil { + return fmt.Errorf("azd init command failed: %w\nOutput: %s", err, string(output)) + } + + return nil +} + +// runAzdUp executes the azd up command +func (s *IntegrationTestSuite) runAzdUp() error { + logf("Running azd up (provisioning Azure resources)...") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + args := []string{"up", "--no-prompt"} + + cmd := exec.CommandContext(ctx, s.azdBinary, args...) + cmd.Dir = s.testEnvDir + + // Capture output for debugging + output, err := cmd.CombinedOutput() + logCommandOutput("azd up", output) + if err != nil { + return fmt.Errorf("azd up command failed: %w\nOutput: %s", err, string(output)) + } + + return nil +} + +// runAzdDown executes the azd down command to clean up Azure resources +func (s *IntegrationTestSuite) runAzdDown() error { + logf("Cleaning up Azure resources...") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + args := []string{"down", "--force", "--purge", "--no-prompt"} + + cmd := exec.CommandContext(ctx, s.azdBinary, args...) + cmd.Dir = s.testEnvDir + + // Capture output for debugging but don't fail cleanup on error + output, err := cmd.CombinedOutput() + logCommandOutput("azd down", output) + if err != nil { + logf("Warning: Azure cleanup failed: %v", err) + fmt.Printf("Warning: azd down command failed during cleanup: %v\nOutput: %s\n", err, string(output)) + } + + return nil // Always return nil to not block cleanup +} + +// retrieveProjectID gets the Azure AI Project ID from azd environment +func (s *IntegrationTestSuite) retrieveProjectID() error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + args := []string{"env", "get-value", "AZURE_AI_PROJECT_ID"} + + cmd := exec.CommandContext(ctx, s.azdBinary, args...) + cmd.Dir = s.testEnvDir + + // Capture output to get the project ID + output, err := cmd.Output() + logCommandOutput("azd env get-value", output) + if err != nil { + return fmt.Errorf("failed to get AZURE_AI_PROJECT_ID from azd env: %w", err) + } + + // Trim whitespace and store the project ID + s.projectID = strings.TrimSpace(string(output)) + if s.projectID == "" { + return fmt.Errorf("AZURE_AI_PROJECT_ID is empty") + } + + return nil +} + +// verifyTestEnvironment ensures the test environment was set up correctly +func (s *IntegrationTestSuite) verifyTestEnvironment() error { + // Check for expected files/directories in the test environment + expectedPaths := []string{ + filepath.Join(s.testEnvDir, "azure.yaml"), + filepath.Join(s.testEnvDir, ".azure"), + } + + for _, path := range expectedPaths { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("expected path %s not found: %w", path, err) + } + } + + return nil +} + +func TestInitCommand_Integration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Ensure test suite is initialized + require.NotNil(t, testSuite, "Test suite should be initialized") + currentTestName = "INIT" + logf("Running integration tests with project ID: %s", testSuite.projectID) + + tests := []struct { + name string + manifestURL string + targetDir string + wantErr bool + }{ + { + name: "InitWithValidManifest", + manifestURL: testManifestURL, + targetDir: "calculator-agent-test", + wantErr: false, + }, + { + name: "InitWithInvalidManifest", + manifestURL: "https://invalid-url.com/agent.yaml", + targetDir: "invalid-agent-test", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + currentTestName = tt.name + logf("Running test: %s", tt.name) + // Create temporary directory for test + tempDir := t.TempDir() + targetPath := filepath.Join(tempDir, tt.targetDir) + logf("Test temp dir: %s", tempDir) + logf("Target path: %s", targetPath) + + // Execute init command + err := executeInitCommand(context.Background(), tt.manifestURL, targetPath) // Assert results + if tt.wantErr { + require.Error(t, err) + logf("Test completed (expected error)") + return + } + + require.NoError(t, err) + + // Verify expected files were created + verifyInitializedProject(t, targetPath) + logf("Test completed successfully") + }) + } +} + +// executeInitCommand executes the AI agent init command with the given parameters +func executeInitCommand(ctx context.Context, manifestURL, targetPath string) error { + // Add timeout to prevent hanging tests + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + // Prepare command arguments + args := []string{"ai", "agent", "init", "--no-prompt"} + + if manifestURL != "" { + args = append(args, "--manifest", manifestURL) + } + + if targetPath != "" { + args = append(args, "--src", targetPath) + } + + // Add project-id retrieved from azd environment + args = append(args, "--project-id", testSuite.projectID) + + logf("Executing command: %s %s", testSuite.azdBinary, strings.Join(args, " ")) + logf("Working directory: %s", testSuite.testEnvDir) + // Execute the command + cmd := exec.CommandContext(ctx, testSuite.azdBinary, args...) + + // Run in the test environment directory where the Azure project is already initialized + cmd.Dir = testSuite.testEnvDir + + // Capture output for debugging + output, err := cmd.CombinedOutput() + logCommandOutput("azd ai agent init", output) + if err != nil { + logf("Command failed with error: %v", err) + return &InitError{ + Message: string(output), + Err: err, + } + } + logf("Command completed successfully") + + // Debug: Show files in working directory as well + if files, err := os.ReadDir(testSuite.testEnvDir); err == nil { + logf("Files in working directory (%s):", testSuite.testEnvDir) + for _, file := range files { + logf(" - %s (dir: %v)", file.Name(), file.IsDir()) + } + } else { + logf("Could not read working directory: %v", err) + } + + return nil +} + +// findAzdBinary locates the azd binary for testing +func findAzdBinary() (string, error) { + // First try to find the binary in the extension's bin directory + extensionRoot := filepath.Join("..", "..", "..") + localBinary := filepath.Join(extensionRoot, "azureaiagent") + if os.PathSeparator == '\\' { + localBinary += ".exe" + } + + if _, err := os.Stat(localBinary); err == nil { + return localBinary, nil + } + + // Fallback to azd in PATH with extension commands + azdPath, err := exec.LookPath("azd") + if err != nil { + return "", &InitError{ + Message: "azd binary not found in PATH and local binary not available", + Err: err, + } + } + + return azdPath, nil +} + +// verifyInitializedProject verifies that the expected files and structure were created +func verifyInitializedProject(t *testing.T, projectPath string) { + t.Helper() + logf("Verifying project at path: %s", projectPath) + + // Verify basic project structure exists + require.DirExists(t, projectPath, "Project directory should exist") + + // List all files in the project directory for debugging + if files, err := os.ReadDir(projectPath); err == nil { + logf("Files in target directory:") + for _, file := range files { + logf(" - %s (dir: %v)", file.Name(), file.IsDir()) + } + } else { + logf("Could not read target directory: %v", err) + } + + // Verify expected files exist in target directory + expectedFilesInTarget := []string{ + "agent.yaml", + "Dockerfile", + "main.py", + "requirements.txt", + } + + for _, file := range expectedFilesInTarget { + fullPath := filepath.Join(projectPath, file) + logf("Checking for file: %s", fullPath) + require.FileExists(t, fullPath, "Expected file %s should exist in target directory", file) + + // Verify file is not empty + info, err := os.Stat(fullPath) + require.NoError(t, err) + require.Greater(t, info.Size(), int64(0), "File %s should not be empty", file) + } + + // Verify azure.yaml was updated in the test environment directory + azureYamlPath := filepath.Join(testSuite.testEnvDir, "azure.yaml") + logf("Checking for azure.yaml in test environment: %s", azureYamlPath) + require.FileExists(t, azureYamlPath, "azure.yaml should exist in test environment directory") + + // Additional verifications can be added here based on your init command's behavior + // For example, checking for infra/ directory, src/ directory, etc. +} + +// InitError represents an error from the init command +type InitError struct { + Message string + Err error +} + +func (e *InitError) Error() string { + if e.Err != nil { + return e.Message + ": " + e.Err.Error() + } + return e.Message +} + +func (e *InitError) Unwrap() error { + return e.Err +} From dc2a181efd798eb64e1996fc2e8417e10e211e2b Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 9 Dec 2025 11:10:14 -0800 Subject: [PATCH 2/7] Add some more tests and separate out utilities Signed-off-by: trangevi --- .../azure.ai.agents/internal/cmd/init.go | 14 +- .../deployTests/deploy_test.go | 302 +++++++++++ .../integrationTests/initTests/init_test.go | 122 +++++ .../test/integrationTests/init_test.go | 510 ------------------ .../testUtilities/test_suite.go | 294 ++++++++++ .../testUtilities/test_utils.go | 256 +++++++++ 6 files changed, 985 insertions(+), 513 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go create mode 100644 cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go delete mode 100644 cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go create mode 100644 cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go create mode 100644 cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index 24afae20330..271caba170e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -971,11 +971,19 @@ func (a *InitAction) downloadAgentYaml( } } - agentId := agentManifest.Name + var agentName string - // Use targetDir if provided or set to local file pointer, otherwise default to "src/{agentId}" + if containerTemplate, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok { + agentName = containerTemplate.Name + } else if promptTemplate, ok := agentManifest.Template.(agent_yaml.PromptAgent); ok { + agentName = promptTemplate.Name + } else { + return nil, "", fmt.Errorf("unsupported agent template type") + } + + // Use targetDir if provided or set to local file pointer, otherwise default to "src/{agentName}" if targetDir == "" { - targetDir = filepath.Join("src", agentId) + targetDir = filepath.Join("src", agentName) } // Create target directory if it doesn't exist diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go new file mode 100644 index 00000000000..87111c380b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package deployTests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" + + "azureaiagent/test/integrationTests/testUtilities" + + "github.com/stretchr/testify/require" +) + +const testManifestURL = "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml" + +// Shared test suite instance for deploy tests +var deployTestSuite *testUtilities.IntegrationTestSuite + +func TestMain(m *testing.M) { + // Initialize logging configuration + testUtilities.InitializeLogging() + + testUtilities.SetCurrentTestName("SETUP") + testUtilities.Logf("Starting deploy test suite") + + // Setup test suite once for all deploy tests + suite, err := testUtilities.SetupTestSuite() + if err != nil { + testUtilities.Logf("Failed to setup test suite: %v", err) + os.Exit(1) + } + deployTestSuite = suite + + // Run tests + code := m.Run() + + // Cleanup + testUtilities.SetCurrentTestName("CLEANUP") + testUtilities.Logf("Running cleanup") + if suite.CleanupFunc != nil { + suite.CleanupFunc() + } + testUtilities.Logf("Deploy test suite completed") + + os.Exit(code) +} + +func TestDeployCommand_Integration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Ensure test suite is initialized + require.NotNil(t, deployTestSuite, "Deploy test suite should be initialized") + testUtilities.SetCurrentTestName("DEPLOY") + testUtilities.Logf("Running integration tests with project ID: %s", deployTestSuite.ProjectID) + + tests := []struct { + name string + agentName string + manifestURL string + wantErr bool + }{ + { + name: "DeployWithValidManifest", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testUtilities.SetCurrentTestName(tt.name) + testUtilities.Logf("Running test: %s", tt.name) + + // Execute init command + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, "", deployTestSuite) + + require.NoError(t, err) + + // Verify expected files were created + testUtilities.VerifyInitializedProject(t, deployTestSuite, "", tt.agentName) + + // Execute deploy command + agentVersion, err := testUtilities.ExecuteDeployCommandForAgent(context.Background(), tt.agentName, deployTestSuite) + if tt.wantErr { + require.Error(t, err) + testUtilities.Logf("Test completed (expected error)") + return + } + + require.NoError(t, err) + if agentVersion != "" { + testUtilities.Logf("Agent deployed with version: %s", agentVersion) + } + + // Wait for agent service to be fully ready after deployment + testUtilities.Logf("Waiting 30 seconds for agent service to initialize...") + time.Sleep(30 * time.Second) + + // Verify deployment was successful + verifyAgentDeployment(t, tt.agentName, agentVersion) + testUtilities.Logf("Test completed successfully") + }) + } +} + +// verifyAgentDeployment checks that the agent was deployed successfully by making API calls +func verifyAgentDeployment(t *testing.T, agentName string, agentVersion string) { + t.Helper() + testUtilities.Logf("Verifying deployment for %s (version: %s)...", agentName, agentVersion) + + // Get required environment variables from azd environment + endpoint, err := deployTestSuite.GetAzdEnvValue("AZURE_AI_PROJECT_ENDPOINT") + require.NoError(t, err, "Failed to get AZURE_AI_PROJECT_ENDPOINT") + require.NotEmpty(t, endpoint, "AZURE_AI_PROJECT_ENDPOINT should be set") + testUtilities.Logf("Using endpoint: %s", endpoint) + + apiVersion := getEnvOrDefault("AGENT_API_VERSION", "2025-05-15-preview") + // Agent version is required - fail if not provided + require.NotEmpty(t, agentVersion, "Agent version should be parsed from deploy command output") + testMessage := getEnvOrDefault("AGENT_TEST_MESSAGE", "What is 2 + 2?") + + // Get Azure access token + token, err := getAzureAccessToken(t) + require.NoError(t, err, "Failed to get Azure access token") + testUtilities.Logf("Successfully obtained Azure access token") + + // Step 1: Create a conversation + conversationID, err := createConversation(t, endpoint, apiVersion, token) + require.NoError(t, err, "Failed to create conversation") + testUtilities.Logf("Created conversation with ID: %s", conversationID) + + // Step 2: Get response from agent + err = testAgentResponse(t, endpoint, apiVersion, token, agentName, agentVersion, testMessage) + require.NoError(t, err, "Failed to get valid response from agent") + + testUtilities.Logf("Deployment verification completed successfully") +} + +// getEnvOrDefault gets an environment variable or returns a default value +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// getAzureAccessToken obtains an Azure access token using az cli +func getAzureAccessToken(t *testing.T) (string, error) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "az", "account", "get-access-token", "--resource", "https://ai.azure.com", "--query", "accessToken", "-o", "tsv") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get access token: %w", err) + } + + token := strings.TrimSpace(string(output)) + if token == "" { + return "", fmt.Errorf("access token is empty") + } + + return token, nil +} + +// createConversation creates a new conversation and returns its ID +func createConversation(t *testing.T, endpoint, apiVersion, token string) (string, error) { + t.Helper() + + conversationURL := fmt.Sprintf("%s/openai/conversations?api-version=%s", endpoint, apiVersion) + + payload := map[string]interface{}{ + "metadata": map[string]string{ + "test_session": "integration_test_agent_response", + }, + } + + payloadBytes, err := json.Marshal(payload) + require.NoError(t, err, "Failed to marshal conversation payload") + + req, err := http.NewRequest("POST", conversationURL, bytes.NewBuffer(payloadBytes)) + require.NoError(t, err, "Failed to create conversation request") + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{Timeout: 2 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("conversation request failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to create conversation (status %d): %s", resp.StatusCode, string(body)) + } + + var conversationData map[string]interface{} + if err := json.Unmarshal(body, &conversationData); err != nil { + return "", fmt.Errorf("failed to parse conversation response: %w", err) + } + + conversationID, ok := conversationData["id"].(string) + if !ok || conversationID == "" { + return "", fmt.Errorf("conversation ID not found in response") + } + + return conversationID, nil +} + +// testAgentResponse sends a test message to the agent and verifies the response +func testAgentResponse(t *testing.T, endpoint, apiVersion, token, agentName, agentVersion, testMessage string) error { + t.Helper() + + requestURL := fmt.Sprintf("%s/openai/responses?api-version=%s", endpoint, apiVersion) + + payload := map[string]interface{}{ + "agent": map[string]string{ + "type": "agent_reference", + "name": agentName, + "version": agentVersion, + }, + "input": testMessage, + } + + payloadBytes, err := json.Marshal(payload) + require.NoError(t, err, "Failed to marshal agent request payload") + + testUtilities.Logf("Agent request payload: %s", string(payloadBytes)) + + req, err := http.NewRequest("POST", requestURL, bytes.NewBuffer(payloadBytes)) + require.NoError(t, err, "Failed to create agent request") + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Set("Content-Type", "application/json") + + // Increase timeout for agent response - agents can take time to process + client := &http.Client{Timeout: 2 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("agent request failed: %w", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get response from agent (status %d): %s", resp.StatusCode, string(body)) + } + + var responseData map[string]interface{} + if err := json.Unmarshal(body, &responseData); err != nil { + return fmt.Errorf("failed to parse agent response: %w", err) + } + + testUtilities.Logf("Agent response data: %s", string(body)) + + // Verify response doesn't contain errors + if errorData, hasError := responseData["error"]; hasError && errorData != nil { + return fmt.Errorf("agent response contains error: %v", errorData) + } + + // Verify response has output + output, hasOutput := responseData["output"] + if !hasOutput { + return fmt.Errorf("response missing 'output' field") + } + + // Check if output is a string or array and verify it's not empty + switch v := output.(type) { + case string: + if len(v) == 0 { + return fmt.Errorf("response output string is empty") + } + testUtilities.Logf("Agent response output (string): %s", v) + case []interface{}: + if len(v) == 0 { + return fmt.Errorf("response output array is empty") + } + testUtilities.Logf("Agent response output (array with %d items)", len(v)) + default: + testUtilities.Logf("Agent response output type: %T", v) + } + + testUtilities.Logf("Agent response validation successful") + return nil +} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go new file mode 100644 index 00000000000..bc1bf9bbe11 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package initTests + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "azureaiagent/test/integrationTests/testUtilities" + + "github.com/stretchr/testify/require" +) + +const testManifestURL = "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml" + +// Shared test suite instance for init tests +var testSuite *testUtilities.IntegrationTestSuite + +// TestMain provides package-level setup and teardown for integration tests +func TestMain(m *testing.M) { + // Initialize logging configuration + testUtilities.InitializeLogging() + + testUtilities.SetCurrentTestName("SETUP") + testUtilities.Logf("Starting integration test suite") + + // Setup + suite, err := testUtilities.SetupTestSuite() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to setup integration test suite: %v\n", err) + os.Exit(1) + } + testSuite = suite + + // Run tests + code := m.Run() + + // Cleanup + testUtilities.SetCurrentTestName("CLEANUP") + testUtilities.Logf("Running cleanup") + if testSuite != nil && testSuite.CleanupFunc != nil { + testSuite.CleanupFunc() + } + testUtilities.Logf("Integration test suite completed") + + os.Exit(code) +} + +func TestInitCommand_Integration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Ensure test suite is initialized + require.NotNil(t, testSuite, "Test suite should be initialized") + testUtilities.SetCurrentTestName("INIT") + testUtilities.Logf("Running integration tests with project ID: %s", testSuite.ProjectID) + + // Create temporary directory for separated src dir test + tempDir := t.TempDir() + + tests := []struct { + name string + agentName string + manifestURL string + targetDir string + wantErr bool + }{ + { + name: "InitWithValidManifestDefaultSrc", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + targetDir: "", + wantErr: false, + }, + { + name: "InitWithValidManifestRelativeSrc", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + targetDir: filepath.Join("src", "calculator-agent-test"), + wantErr: false, + }, + { + name: "InitWithValidManifestExternalSrc", + agentName: "CalculatorAgentLG", + manifestURL: testManifestURL, + targetDir: filepath.Join(tempDir, "calculator-agent-test"), + wantErr: false, + }, + { + name: "InitWithInvalidManifest", + manifestURL: "https://invalid-url.com/agent.yaml", + targetDir: "invalid-agent-test", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testUtilities.SetCurrentTestName(tt.name) + testUtilities.Logf("Running test: %s", tt.name) + + // Execute init command + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, tt.targetDir, testSuite) + if tt.wantErr { + require.Error(t, err) + testUtilities.Logf("Test completed (expected error)") + return + } + + require.NoError(t, err) + + // Verify expected files were created + testUtilities.VerifyInitializedProject(t, testSuite, tt.targetDir, tt.agentName) + testUtilities.Logf("Test completed successfully") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go deleted file mode 100644 index c1f2b21c5bf..00000000000 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/init_test.go +++ /dev/null @@ -1,510 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package integrationTests - -import ( - "context" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -const testManifestURL = "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/microsoft/python/getting-started-agents/hosted-agents/calculator-agent/agent.yaml" - -var ( - // Global test state - testSuite *IntegrationTestSuite - // Verbose logging flag - verboseLogging bool - // Current test context for logging - currentTestName string -) - -// logf logs a message if verbose logging is enabled -func logf(format string, args ...interface{}) { - if verboseLogging { - prefix := "[INTEGRATION]" - if currentTestName != "" { - prefix = fmt.Sprintf("[%s]", currentTestName) - } - log.Printf(prefix+" "+format, args...) - } -} - -// logCommandOutput logs command output if verbose logging is enabled -func logCommandOutput(cmd string, output []byte) { - if verboseLogging && len(output) > 0 { - log.Printf("[COMMAND] %s output:\n%s", cmd, string(output)) - } -} - -// IntegrationTestSuite holds shared test resources and state -type IntegrationTestSuite struct { - azdBinary string - testEnvDir string - cleanupFunc func() - projectID string // Retrieved from azd env after provisioning -} - -// TestMain provides package-level setup and teardown for integration tests -func TestMain(m *testing.M) { - // Check if verbose logging is enabled by looking at command line args - for _, arg := range os.Args { - if arg == "-v" || arg == "-test.v" || arg == "-test.v=true" { - verboseLogging = true - break - } - } - if verboseLogging { - log.SetFlags(log.LstdFlags | log.Lshortfile) - } - - currentTestName = "SETUP" - logf("Starting integration test suite") - - // Setup - suite, err := setupIntegrationTestSuite() - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to setup integration test suite: %v\n", err) - os.Exit(1) - } - testSuite = suite - - // Run tests - code := m.Run() - - // Cleanup - currentTestName = "CLEANUP" - logf("Running cleanup") - if testSuite != nil && testSuite.cleanupFunc != nil { - testSuite.cleanupFunc() - } - logf("Integration test suite completed") - - os.Exit(code) -} - -// setupIntegrationTestSuite initializes shared test resources -func setupIntegrationTestSuite() (*IntegrationTestSuite, error) { - logf("Setting up integration test suite") - suite := &IntegrationTestSuite{} - - // Find azd binary - azdPath, err := findAzdBinary() - if err != nil { - return nil, fmt.Errorf("failed to find azd binary: %w", err) - } - suite.azdBinary = azdPath - logf("Found azd binary: %s", azdPath) - - // Verify azd binary works - if err := suite.verifyAzdBinary(); err != nil { - return nil, fmt.Errorf("azd binary verification failed: %w", err) - } - - // Create test environment directory - testEnvDir, err := os.MkdirTemp("", "azd-ai-agents-test-env-*") - if err != nil { - return nil, fmt.Errorf("failed to create test environment directory: %w", err) - } - suite.testEnvDir = testEnvDir - logf("Created test environment: %s", testEnvDir) - suite.cleanupFunc = func() { - // Run azd down to clean up Azure resources - suite.runAzdDown() - // Remove local test directory - if err := os.RemoveAll(testEnvDir); err != nil { - logf("Warning: failed to remove test directory: %v", err) - } - } - - // Initialize test environment with azd template - logf("Initializing Azure environment...") - if err := suite.initializeTestEnvironment(); err != nil { - suite.cleanupFunc() - return nil, fmt.Errorf("failed to initialize test environment: %w", err) - } - logf("Azure environment ready (Project ID: %s)", suite.projectID) - - return suite, nil -} - -// verifyAzdBinary ensures the azd binary is working and has the ai agent extension -func (s *IntegrationTestSuite) verifyAzdBinary() error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // Test basic azd functionality - cmd := exec.CommandContext(ctx, s.azdBinary, "version") - output, err := cmd.CombinedOutput() - logCommandOutput("azd version", output) - if err != nil { - return fmt.Errorf("azd version command failed: %w", err) - } - - // Test ai agent extension is available - cmd = exec.CommandContext(ctx, s.azdBinary, "ai", "agent", "--help") - output, err = cmd.CombinedOutput() - logCommandOutput("azd ai agent --help", output) - if err != nil { - return fmt.Errorf("azd ai agent extension not available: %w", err) - } - - return nil -} - -// initializeTestEnvironment sets up the test environment with azd template and provisions resources -func (s *IntegrationTestSuite) initializeTestEnvironment() error { - // Change to test environment directory once - originalDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - - if err := os.Chdir(s.testEnvDir); err != nil { - return fmt.Errorf("failed to change to test environment directory: %w", err) - } - defer os.Chdir(originalDir) - - // Run azd init command - if err := s.runAzdInit(); err != nil { - return fmt.Errorf("failed to run azd init: %w", err) - } - - // Verify the environment was created successfully - if err := s.verifyTestEnvironment(); err != nil { - return fmt.Errorf("test environment verification failed: %w", err) - } - - // Run azd up to provision Azure resources - if err := s.runAzdUp(); err != nil { - return fmt.Errorf("failed to run azd up: %w", err) - } - - // Retrieve project ID from azd environment - if err := s.retrieveProjectID(); err != nil { - return fmt.Errorf("failed to retrieve project ID: %w", err) - } - - return nil -} - -// runAzdInit executes the azd init command -func (s *IntegrationTestSuite) runAzdInit() error { - logf("Running azd init...") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - args := []string{ - "init", - "-t", "Azure-Samples/azd-ai-starter-basic", - "-e", "trangevi-test", - "--location", "westus2", - "--subscription", "827cb315-a120-4b3d-bd80-93f7b3126af2", - "--no-prompt", - } - - cmd := exec.CommandContext(ctx, s.azdBinary, args...) - cmd.Dir = s.testEnvDir - - // Capture output for debugging - output, err := cmd.CombinedOutput() - logCommandOutput("azd init", output) - if err != nil { - return fmt.Errorf("azd init command failed: %w\nOutput: %s", err, string(output)) - } - - return nil -} - -// runAzdUp executes the azd up command -func (s *IntegrationTestSuite) runAzdUp() error { - logf("Running azd up (provisioning Azure resources)...") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - args := []string{"up", "--no-prompt"} - - cmd := exec.CommandContext(ctx, s.azdBinary, args...) - cmd.Dir = s.testEnvDir - - // Capture output for debugging - output, err := cmd.CombinedOutput() - logCommandOutput("azd up", output) - if err != nil { - return fmt.Errorf("azd up command failed: %w\nOutput: %s", err, string(output)) - } - - return nil -} - -// runAzdDown executes the azd down command to clean up Azure resources -func (s *IntegrationTestSuite) runAzdDown() error { - logf("Cleaning up Azure resources...") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - args := []string{"down", "--force", "--purge", "--no-prompt"} - - cmd := exec.CommandContext(ctx, s.azdBinary, args...) - cmd.Dir = s.testEnvDir - - // Capture output for debugging but don't fail cleanup on error - output, err := cmd.CombinedOutput() - logCommandOutput("azd down", output) - if err != nil { - logf("Warning: Azure cleanup failed: %v", err) - fmt.Printf("Warning: azd down command failed during cleanup: %v\nOutput: %s\n", err, string(output)) - } - - return nil // Always return nil to not block cleanup -} - -// retrieveProjectID gets the Azure AI Project ID from azd environment -func (s *IntegrationTestSuite) retrieveProjectID() error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - args := []string{"env", "get-value", "AZURE_AI_PROJECT_ID"} - - cmd := exec.CommandContext(ctx, s.azdBinary, args...) - cmd.Dir = s.testEnvDir - - // Capture output to get the project ID - output, err := cmd.Output() - logCommandOutput("azd env get-value", output) - if err != nil { - return fmt.Errorf("failed to get AZURE_AI_PROJECT_ID from azd env: %w", err) - } - - // Trim whitespace and store the project ID - s.projectID = strings.TrimSpace(string(output)) - if s.projectID == "" { - return fmt.Errorf("AZURE_AI_PROJECT_ID is empty") - } - - return nil -} - -// verifyTestEnvironment ensures the test environment was set up correctly -func (s *IntegrationTestSuite) verifyTestEnvironment() error { - // Check for expected files/directories in the test environment - expectedPaths := []string{ - filepath.Join(s.testEnvDir, "azure.yaml"), - filepath.Join(s.testEnvDir, ".azure"), - } - - for _, path := range expectedPaths { - if _, err := os.Stat(path); err != nil { - return fmt.Errorf("expected path %s not found: %w", path, err) - } - } - - return nil -} - -func TestInitCommand_Integration(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - // Ensure test suite is initialized - require.NotNil(t, testSuite, "Test suite should be initialized") - currentTestName = "INIT" - logf("Running integration tests with project ID: %s", testSuite.projectID) - - tests := []struct { - name string - manifestURL string - targetDir string - wantErr bool - }{ - { - name: "InitWithValidManifest", - manifestURL: testManifestURL, - targetDir: "calculator-agent-test", - wantErr: false, - }, - { - name: "InitWithInvalidManifest", - manifestURL: "https://invalid-url.com/agent.yaml", - targetDir: "invalid-agent-test", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - currentTestName = tt.name - logf("Running test: %s", tt.name) - // Create temporary directory for test - tempDir := t.TempDir() - targetPath := filepath.Join(tempDir, tt.targetDir) - logf("Test temp dir: %s", tempDir) - logf("Target path: %s", targetPath) - - // Execute init command - err := executeInitCommand(context.Background(), tt.manifestURL, targetPath) // Assert results - if tt.wantErr { - require.Error(t, err) - logf("Test completed (expected error)") - return - } - - require.NoError(t, err) - - // Verify expected files were created - verifyInitializedProject(t, targetPath) - logf("Test completed successfully") - }) - } -} - -// executeInitCommand executes the AI agent init command with the given parameters -func executeInitCommand(ctx context.Context, manifestURL, targetPath string) error { - // Add timeout to prevent hanging tests - ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) - defer cancel() - - // Prepare command arguments - args := []string{"ai", "agent", "init", "--no-prompt"} - - if manifestURL != "" { - args = append(args, "--manifest", manifestURL) - } - - if targetPath != "" { - args = append(args, "--src", targetPath) - } - - // Add project-id retrieved from azd environment - args = append(args, "--project-id", testSuite.projectID) - - logf("Executing command: %s %s", testSuite.azdBinary, strings.Join(args, " ")) - logf("Working directory: %s", testSuite.testEnvDir) - // Execute the command - cmd := exec.CommandContext(ctx, testSuite.azdBinary, args...) - - // Run in the test environment directory where the Azure project is already initialized - cmd.Dir = testSuite.testEnvDir - - // Capture output for debugging - output, err := cmd.CombinedOutput() - logCommandOutput("azd ai agent init", output) - if err != nil { - logf("Command failed with error: %v", err) - return &InitError{ - Message: string(output), - Err: err, - } - } - logf("Command completed successfully") - - // Debug: Show files in working directory as well - if files, err := os.ReadDir(testSuite.testEnvDir); err == nil { - logf("Files in working directory (%s):", testSuite.testEnvDir) - for _, file := range files { - logf(" - %s (dir: %v)", file.Name(), file.IsDir()) - } - } else { - logf("Could not read working directory: %v", err) - } - - return nil -} - -// findAzdBinary locates the azd binary for testing -func findAzdBinary() (string, error) { - // First try to find the binary in the extension's bin directory - extensionRoot := filepath.Join("..", "..", "..") - localBinary := filepath.Join(extensionRoot, "azureaiagent") - if os.PathSeparator == '\\' { - localBinary += ".exe" - } - - if _, err := os.Stat(localBinary); err == nil { - return localBinary, nil - } - - // Fallback to azd in PATH with extension commands - azdPath, err := exec.LookPath("azd") - if err != nil { - return "", &InitError{ - Message: "azd binary not found in PATH and local binary not available", - Err: err, - } - } - - return azdPath, nil -} - -// verifyInitializedProject verifies that the expected files and structure were created -func verifyInitializedProject(t *testing.T, projectPath string) { - t.Helper() - logf("Verifying project at path: %s", projectPath) - - // Verify basic project structure exists - require.DirExists(t, projectPath, "Project directory should exist") - - // List all files in the project directory for debugging - if files, err := os.ReadDir(projectPath); err == nil { - logf("Files in target directory:") - for _, file := range files { - logf(" - %s (dir: %v)", file.Name(), file.IsDir()) - } - } else { - logf("Could not read target directory: %v", err) - } - - // Verify expected files exist in target directory - expectedFilesInTarget := []string{ - "agent.yaml", - "Dockerfile", - "main.py", - "requirements.txt", - } - - for _, file := range expectedFilesInTarget { - fullPath := filepath.Join(projectPath, file) - logf("Checking for file: %s", fullPath) - require.FileExists(t, fullPath, "Expected file %s should exist in target directory", file) - - // Verify file is not empty - info, err := os.Stat(fullPath) - require.NoError(t, err) - require.Greater(t, info.Size(), int64(0), "File %s should not be empty", file) - } - - // Verify azure.yaml was updated in the test environment directory - azureYamlPath := filepath.Join(testSuite.testEnvDir, "azure.yaml") - logf("Checking for azure.yaml in test environment: %s", azureYamlPath) - require.FileExists(t, azureYamlPath, "azure.yaml should exist in test environment directory") - - // Additional verifications can be added here based on your init command's behavior - // For example, checking for infra/ directory, src/ directory, etc. -} - -// InitError represents an error from the init command -type InitError struct { - Message string - Err error -} - -func (e *InitError) Error() string { - if e.Err != nil { - return e.Message + ": " + e.Err.Error() - } - return e.Message -} - -func (e *InitError) Unwrap() error { - return e.Err -} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go new file mode 100644 index 00000000000..f872bd99869 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go @@ -0,0 +1,294 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package testUtilities + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" +) + +// IntegrationTestSuite holds shared test resources and state +type IntegrationTestSuite struct { + AzdBinary string + AzdProjectDir string + CleanupFunc func() + ProjectID string // Retrieved from azd env after provisioning +} + +// SetupTestSuite initializes shared test resources +func SetupTestSuite() (*IntegrationTestSuite, error) { + Logf("Setting up test suite") + suite := &IntegrationTestSuite{} + + // Find azd binary + azdPath, err := findAzdBinary() + if err != nil { + return nil, fmt.Errorf("failed to find azd binary: %w", err) + } + suite.AzdBinary = azdPath + Logf("Found azd binary: %s", azdPath) + + // Verify azd binary works + if err := suite.verifyAzdBinary(); err != nil { + return nil, fmt.Errorf("azd binary verification failed: %w", err) + } + + // Create test environment directory + azdProjectDir, err := os.MkdirTemp("", "azd-ai-agents-test-env-*") + if err != nil { + return nil, fmt.Errorf("failed to create test environment directory: %w", err) + } + suite.AzdProjectDir = azdProjectDir + Logf("Created test environment: %s", azdProjectDir) + suite.CleanupFunc = func() { + // Run azd down to clean up Azure resources + suite.runAzdDown() + // Remove local test directory + if err := os.RemoveAll(azdProjectDir); err != nil { + Logf("Warning: failed to remove test directory: %v", err) + } + } + + // Initialize test environment with azd template + Logf("Initializing Azure environment...") + if err := suite.initializeAzdProject(); err != nil { + suite.CleanupFunc() + return nil, fmt.Errorf("failed to initialize test environment: %w", err) + } + Logf("Azure environment ready (Project ID: %s)", suite.ProjectID) + + return suite, nil +} + +// verifyAzdBinary ensures the azd binary is working and has the ai agent extension +func (s *IntegrationTestSuite) verifyAzdBinary() error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Test basic azd functionality + cmd := exec.CommandContext(ctx, s.AzdBinary, "version") + var outputBuilder strings.Builder + cmd.Stdout = &outputBuilder + cmd.Stderr = &outputBuilder + err := cmd.Run() + output := outputBuilder.String() + LogCommandOutput("azd version", []byte(output)) + if err != nil { + return fmt.Errorf("azd version command failed: %w", err) + } + + // Test ai agent extension is available + cmd = exec.CommandContext(ctx, s.AzdBinary, "ai", "agent", "--help") + outputBuilder.Reset() + cmd.Stdout = &outputBuilder + cmd.Stderr = &outputBuilder + err = cmd.Run() + output = outputBuilder.String() + LogCommandOutput("azd ai agent --help", []byte(output)) + if err != nil { + return fmt.Errorf("azd ai agent extension not available: %w", err) + } + + return nil +} + +// initializeAzdProject sets up the test environment with azd template and provisions resources +func (s *IntegrationTestSuite) initializeAzdProject() error { + // Change to test environment directory once + originalDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + + if err := os.Chdir(s.AzdProjectDir); err != nil { + return fmt.Errorf("failed to change to test environment directory: %w", err) + } + defer os.Chdir(originalDir) + + // Run azd init command + if err := s.runAzdInit(true); err != nil { + return fmt.Errorf("failed to run azd init: %w", err) + } + + // Verify the environment was created successfully + if err := s.verifyTestEnvironment(); err != nil { + return fmt.Errorf("test environment verification failed: %w", err) + } + + // Set required environment variable for provisioning an ACR + if err := s.SetAzdEnvValue("ENABLE_HOSTED_AGENTS", "true"); err != nil { + return fmt.Errorf("failed to set ENABLE_HOSTED_AGENTS: %w", err) + } + + // Initialize a calculator agent into the project so we have the model we need + if err := ExecuteInitCommandForAgent(context.Background(), + "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml", + "", s); err != nil { + return fmt.Errorf("failed to initialize calculator agent: %w", err) + } + + // Run azd up to provision Azure resources + if err := s.runAzdUp(); err != nil { + return fmt.Errorf("failed to run azd up: %w", err) + } + + // Retrieve project ID from azd environment + projectID, err := s.GetAzdEnvValue("AZURE_AI_PROJECT_ID") + if err != nil { + return fmt.Errorf("failed to retrieve project ID: %w", err) + } + if projectID == "" { + return fmt.Errorf("AZURE_AI_PROJECT_ID is empty") + } + s.ProjectID = projectID + + return nil +} + +// runAzdInit executes the azd init command +func (s *IntegrationTestSuite) runAzdInit(withTemplate bool) error { + Logf("Running azd init...") + + // Generate a unique environment name using a short UUID suffix + suffix := uuid.New().String()[:8] + + args := []string{ + "init", + "-e", fmt.Sprintf("azd-extension-integration-tests-%s", suffix), + "--location", "northcentralus", + "--subscription", "827cb315-a120-4b3d-bd80-93f7b3126af2", + "--no-prompt", + } + + if withTemplate { + args = append(args, "-t", "Azure-Samples/azd-ai-starter-basic") + } + + _, err := executeAzdCommand(context.Background(), s, 5*time.Minute, args) + if err != nil { + return fmt.Errorf("azd init command failed: %w", err) + } + + return nil +} + +// runAzdUp executes the azd up command +func (s *IntegrationTestSuite) runAzdUp() error { + Logf("Running azd up (provisioning Azure resources)...") + + args := []string{"up", "--no-prompt"} + + _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) + if err != nil { + return fmt.Errorf("azd up command failed: %w", err) + } + + return nil +} + +// runAzdDown executes the azd down command to clean up Azure resources +func (s *IntegrationTestSuite) runAzdDown() error { + Logf("Cleaning up Azure resources...") + + args := []string{"down", "--force", "--purge", "--no-prompt"} + + _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) + if err != nil { + Logf("Warning: Azure cleanup failed: %v", err) + } + + return nil // Always return nil to not block cleanup +} + +// GetAzdEnvValue retrieves an environment variable value from the azd environment +func (s *IntegrationTestSuite) GetAzdEnvValue(key string) (string, error) { + Logf("Running azd env get-value %s...", key) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + args := []string{"env", "get-value", key} + + cmd := exec.CommandContext(ctx, s.AzdBinary, args...) + cmd.Dir = s.AzdProjectDir + + // Capture output to get the value + output, err := cmd.Output() + LogCommandOutput(fmt.Sprintf("azd env get-value %s", key), output) + if err != nil { + return "", fmt.Errorf("failed to get %s from azd env: %w", key, err) + } + + // Trim whitespace and return the value + return strings.TrimSpace(string(output)), nil +} + +// SetAzdEnvValue sets an environment variable value in the azd environment +func (s *IntegrationTestSuite) SetAzdEnvValue(key string, value string) error { + Logf("Running azd env set %s %s...", key, value) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + args := []string{"env", "set", key, value} + + cmd := exec.CommandContext(ctx, s.AzdBinary, args...) + cmd.Dir = s.AzdProjectDir + + // Capture output to get the value + output, err := cmd.Output() + LogCommandOutput(fmt.Sprintf("azd env set %s", key), output) + if err != nil { + return fmt.Errorf("failed to set %s in azd env: %w", key, err) + } + + return nil +} + +// verifyTestEnvironment ensures the test environment was set up correctly +func (s *IntegrationTestSuite) verifyTestEnvironment() error { + // Check for expected files/directories in the test environment + expectedFiles := []string{ + filepath.Join(s.AzdProjectDir, "azure.yaml"), + filepath.Join(s.AzdProjectDir, ".azure"), + } + + for _, path := range expectedFiles { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("expected path %s not found: %w", path, err) + } + } + + return nil +} + +// findAzdBinary locates the azd binary for testing +func findAzdBinary() (string, error) { + // First try to find azd in PATH (the main CLI with extension support) + azdPath, err := exec.LookPath("azd") + if err == nil { + return azdPath, nil + } + + // Fallback to local binary for development scenarios + extensionRoot := filepath.Join("..", "..", "..") + localBinary := filepath.Join(extensionRoot, "azureaiagent") + if os.PathSeparator == '\\' { + localBinary += ".exe" + } + + if _, err := os.Stat(localBinary); err == nil { + return localBinary, nil + } + + return "", &InitError{ + Message: "azd binary not found in PATH and local binary not available", + Err: err, + } +} diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go new file mode 100644 index 00000000000..7bf6ce9ec1e --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package testUtilities + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + // Verbose logging flag + verboseLogging bool + // Current test context for logging + currentTestName string +) + +// InitializeLogging sets up logging configuration for integration tests +func InitializeLogging() { + // Check if verbose logging is enabled by looking at command line args + for _, arg := range os.Args { + if arg == "-v" || arg == "-test.v" || arg == "-test.v=true" { + verboseLogging = true + break + } + } + if verboseLogging { + log.SetFlags(log.LstdFlags | log.Lshortfile) + } +} + +// Logf logs a message if verbose logging is enabled +func Logf(format string, args ...interface{}) { + if verboseLogging { + prefix := "[INTEGRATION]" + if currentTestName != "" { + prefix = fmt.Sprintf("[%s]", currentTestName) + } + log.Printf(prefix+" "+format, args...) + } +} + +// LogCommandOutput logs command output if verbose logging is enabled +func LogCommandOutput(cmd string, output []byte) { + if verboseLogging && len(output) > 0 { + log.Printf("[COMMAND] %s output:\n%s", cmd, string(output)) + } +} + +// SetCurrentTestName sets the current test name for logging context +func SetCurrentTestName(name string) { + currentTestName = name +} + +// ExecuteInitCommandForAgent executes the AI agent init command with the given parameters +func ExecuteInitCommandForAgent(ctx context.Context, manifestURL, targetPath string, testSuite *IntegrationTestSuite) error { + // Prepare command arguments + args := []string{"ai", "agent", "init", "--no-prompt"} + + if manifestURL != "" { + args = append(args, "--manifest", manifestURL) + } + + if targetPath != "" { + args = append(args, "--src", targetPath) + } + + // Add project-id retrieved from azd environment + if testSuite.ProjectID != "" { + args = append(args, "--project-id", testSuite.ProjectID) + } + + _, err := executeAzdCommand(ctx, testSuite, 2*time.Minute, args) + if err != nil { + return err + } + + // Debug: Show files in working directory as well + if files, err := os.ReadDir(testSuite.AzdProjectDir); err == nil { + Logf("Files in working directory (%s):", testSuite.AzdProjectDir) + for _, file := range files { + Logf(" - %s (dir: %v)", file.Name(), file.IsDir()) + } + } else { + Logf("Could not read working directory: %v", err) + } + + return nil +} + +// VerifyInitializedProject verifies that the expected files and structure were created +func VerifyInitializedProject(t *testing.T, testSuite *IntegrationTestSuite, srcDir string, agentName string) { + t.Helper() + if srcDir == "" { + srcDir = filepath.Join(testSuite.AzdProjectDir, "src", agentName) + } else if !filepath.IsAbs(srcDir) { + // If srcDir is relative, make it relative to the azd project directory + srcDir = filepath.Join(testSuite.AzdProjectDir, srcDir) + } + Logf("Verifying project at path: %s", srcDir) + + // Verify basic project structure exists + require.DirExists(t, srcDir, "Project directory should exist") + + // List all files in the project directory for debugging + if files, err := os.ReadDir(srcDir); err == nil { + Logf("Files in target directory:") + for _, file := range files { + Logf(" - %s (dir: %v)", file.Name(), file.IsDir()) + } + } else { + Logf("Could not read target directory: %v", err) + } + + // Verify expected files exist in target directory + expectedFilesInTarget := []string{ + "agent.yaml", + "Dockerfile", + "main.py", + "requirements.txt", + } + + for _, file := range expectedFilesInTarget { + fullPath := filepath.Join(srcDir, file) + Logf("Checking for file: %s", fullPath) + require.FileExists(t, fullPath, "Expected file %s should exist in target directory", file) + + // Verify file is not empty + info, err := os.Stat(fullPath) + require.NoError(t, err) + require.Greater(t, info.Size(), int64(0), "File %s should not be empty", file) + } + + // Verify azure.yaml was updated in the test environment + azureYamlPath := filepath.Join(testSuite.AzdProjectDir, "azure.yaml") + Logf("Checking for azure.yaml in test environment: %s", azureYamlPath) + require.FileExists(t, azureYamlPath, "azure.yaml should exist in test environment directory") + + // Additional verifications can be added here based on your init command's behavior + // For example, checking for infra/ directory, src/ directory, etc. +} + +// InitError represents an error from the init command +type InitError struct { + Message string + Err error +} + +func (e *InitError) Error() string { + if e.Err != nil { + return e.Message + ": " + e.Err.Error() + } + return e.Message +} + +func (e *InitError) Unwrap() error { + return e.Err +} + +// executeAzdCommand executes an azd command and returns output and agent version if available +func executeAzdCommand(ctx context.Context, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + Logf("Executing command: %s %s", testSuite.AzdBinary, strings.Join(args, " ")) + Logf("Working directory: %s", testSuite.AzdProjectDir) + + // Execute the command + cmd := exec.CommandContext(ctx, testSuite.AzdBinary, args...) + cmd.Dir = testSuite.AzdProjectDir + + // Use a strings.Builder to capture output without truncation + var outputBuilder strings.Builder + cmd.Stdout = &outputBuilder + cmd.Stderr = &outputBuilder + + // Run the command + err := cmd.Run() + output := outputBuilder.String() + + // Log the full output + LogCommandOutput(strings.Join(args, " "), []byte(output)) + + if err != nil { + Logf("Command failed with error: %v", err) + return "", &InitError{ + Message: output, + Err: err, + } + } + Logf("Command completed successfully") + + return output, nil +} + +func parseOutputForAgentVersion(output string) string { + // Parse the agent version from the output + // Look for pattern: "Agent endpoint: .../agents/{agentName}/versions/{version}" + versionRegex := regexp.MustCompile(`Agent endpoint:.*?/agents/[^/]+/versions/(\d+)`) + matches := versionRegex.FindStringSubmatch(output) + + var agentVersion string + if len(matches) > 1 { + agentVersion = matches[1] + Logf("Parsed agent version: %s", agentVersion) + } else { + Logf("Warning: Could not parse agent version from output") + } + + return agentVersion +} + +// ExecuteUpCommandForAgent executes the AZD up command with the given parameters +// Returns the deployed agent version number if successful +func ExecuteUpCommandForAgent(ctx context.Context, testSuite *IntegrationTestSuite) (string, error) { + args := []string{"up", "--no-prompt"} + + output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) + if err != nil { + return "", err + } + + agentVersion := parseOutputForAgentVersion(output) + + return agentVersion, nil +} + +// ExecuteDeployCommandForAgent executes the AZD deploy command with the given parameters +// Returns the deployed agent version number if successful +func ExecuteDeployCommandForAgent(ctx context.Context, agentName string, testSuite *IntegrationTestSuite) (string, error) { + // Prepare command arguments + args := []string{"deploy"} + if agentName != "" { + args = append(args, agentName) + } + args = append(args, "--no-prompt") + + output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) + if err != nil { + return "", err + } + + agentVersion := parseOutputForAgentVersion(output) + + return agentVersion, nil +} From f91db64d576dbcfb9f5a085038ba80eca7b61387 Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 16 Dec 2025 16:29:55 -0800 Subject: [PATCH 3/7] Some changes for test recording Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/go.mod | 5 +- cli/azd/extensions/azure.ai.agents/go.sum | 8 ++++ .../deployTests/deploy_test.go | 11 ++++- .../integrationTests/initTests/init_test.go | 5 +- .../testUtilities/test_suite.go | 10 ++-- .../testUtilities/test_utils.go | 48 ++++++++++++++++--- 6 files changed, 72 insertions(+), 15 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index b499256b2f8..bf11eec183a 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -17,6 +17,7 @@ require ( github.com/google/uuid v1.6.0 github.com/mark3labs/mcp-go v0.41.1 github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 @@ -26,7 +27,9 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v3 v3.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect @@ -76,7 +79,6 @@ require ( github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -96,6 +98,7 @@ require ( golang.org/x/text v0.31.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251007200510-49b9836ed3ff // indirect google.golang.org/grpc v1.76.0 // indirect + gopkg.in/dnaeon/go-vcr.v3 v3.2.0 // indirect ) replace github.com/azure/azure-dev/cli/azd => ../.. diff --git a/cli/azd/extensions/azure.ai.agents/go.sum b/cli/azd/extensions/azure.ai.agents/go.sum index 5460de3694f..8a44b337162 100644 --- a/cli/azd/extensions/azure.ai.agents/go.sum +++ b/cli/azd/extensions/azure.ai.agents/go.sum @@ -13,6 +13,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDo github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers v1.1.0 h1:fdAOz6TFldGDoEcRa975i5L5QvWU8ptut+SJAIfuWUY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers v1.1.0/go.mod h1:qV+BWew22CAalRTwJEAHs+aSLP49k/csNlspqhMIDRU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v3 v3.1.0 h1:ilMZ576u8sm975EqV+AKEtD4u9TLwqEo2XY9csPXBRo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v3 v3.1.0/go.mod h1:LGhzy+pg9AKr1Z7ZRyTC1qr1xNyVqLsqydvLdY+2iQk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= @@ -21,6 +23,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2/go.mod h1:jVRrRDLCOuif95HDYC23ADTMlvahB7tMdl519m9Iyjc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cognitiveservices/armcognitiveservices v1.8.0 h1:ZMGAqCZov8+7iFUPWKVcTaLgNXUeTlz20sIuWkQWNfg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cognitiveservices/armcognitiveservices v1.8.0/go.mod h1:BElPQ/GZtrdQ2i5uDZw3OKLE1we75W0AEWyeBR1TWQA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= @@ -29,6 +33,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= @@ -296,6 +302,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= +gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go index 87111c380b3..a3185c9be6a 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go @@ -18,6 +18,8 @@ import ( "azureaiagent/test/integrationTests/testUtilities" + "github.com/azure/azure-dev/cli/azd/test/azdcli" + "github.com/azure/azure-dev/cli/azd/test/recording" "github.com/stretchr/testify/require" ) @@ -84,8 +86,13 @@ func TestDeployCommand_Integration(t *testing.T) { testUtilities.SetCurrentTestName(tt.name) testUtilities.Logf("Running test: %s", tt.name) + session := recording.Start(t) + + cli := azdcli.NewCLI(t, azdcli.WithSession(session)) + cli.WorkingDirectory = deployTestSuite.AzdProjectDir + // Execute init command - err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, "", deployTestSuite) + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), cli, tt.manifestURL, "", deployTestSuite) require.NoError(t, err) @@ -93,7 +100,7 @@ func TestDeployCommand_Integration(t *testing.T) { testUtilities.VerifyInitializedProject(t, deployTestSuite, "", tt.agentName) // Execute deploy command - agentVersion, err := testUtilities.ExecuteDeployCommandForAgent(context.Background(), tt.agentName, deployTestSuite) + agentVersion, err := testUtilities.ExecuteDeployCommandForAgent(context.Background(), cli, tt.agentName, deployTestSuite) if tt.wantErr { require.Error(t, err) testUtilities.Logf("Test completed (expected error)") diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go index bc1bf9bbe11..2786efaa915 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go @@ -12,6 +12,7 @@ import ( "azureaiagent/test/integrationTests/testUtilities" + "github.com/azure/azure-dev/cli/azd/test/azdcli" "github.com/stretchr/testify/require" ) @@ -104,8 +105,10 @@ func TestInitCommand_Integration(t *testing.T) { testUtilities.SetCurrentTestName(tt.name) testUtilities.Logf("Running test: %s", tt.name) + cli := azdcli.NewCLI(t) + // Execute init command - err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, tt.targetDir, testSuite) + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), cli, tt.manifestURL, tt.targetDir, testSuite) if tt.wantErr { require.Error(t, err) testUtilities.Logf("Test completed (expected error)") diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go index f872bd99869..51e01ad7ec9 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go @@ -129,7 +129,9 @@ func (s *IntegrationTestSuite) initializeAzdProject() error { } // Initialize a calculator agent into the project so we have the model we need - if err := ExecuteInitCommandForAgent(context.Background(), + if err := ExecuteInitCommandForAgent( + context.Background(), + nil, "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml", "", s); err != nil { return fmt.Errorf("failed to initialize calculator agent: %w", err) @@ -172,7 +174,7 @@ func (s *IntegrationTestSuite) runAzdInit(withTemplate bool) error { args = append(args, "-t", "Azure-Samples/azd-ai-starter-basic") } - _, err := executeAzdCommand(context.Background(), s, 5*time.Minute, args) + _, err := executeAzdCommandWithExec(context.Background(), s, 5*time.Minute, args) if err != nil { return fmt.Errorf("azd init command failed: %w", err) } @@ -186,7 +188,7 @@ func (s *IntegrationTestSuite) runAzdUp() error { args := []string{"up", "--no-prompt"} - _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) + _, err := executeAzdCommandWithExec(context.Background(), s, 10*time.Minute, args) if err != nil { return fmt.Errorf("azd up command failed: %w", err) } @@ -200,7 +202,7 @@ func (s *IntegrationTestSuite) runAzdDown() error { args := []string{"down", "--force", "--purge", "--no-prompt"} - _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) + _, err := executeAzdCommandWithExec(context.Background(), s, 10*time.Minute, args) if err != nil { Logf("Warning: Azure cleanup failed: %v", err) } diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go index 7bf6ce9ec1e..d813f6fff00 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/azure/azure-dev/cli/azd/test/azdcli" "github.com/stretchr/testify/require" ) @@ -63,7 +64,7 @@ func SetCurrentTestName(name string) { } // ExecuteInitCommandForAgent executes the AI agent init command with the given parameters -func ExecuteInitCommandForAgent(ctx context.Context, manifestURL, targetPath string, testSuite *IntegrationTestSuite) error { +func ExecuteInitCommandForAgent(ctx context.Context, cli *azdcli.CLI, manifestURL, targetPath string, testSuite *IntegrationTestSuite) error { // Prepare command arguments args := []string{"ai", "agent", "init", "--no-prompt"} @@ -80,7 +81,14 @@ func ExecuteInitCommandForAgent(ctx context.Context, manifestURL, targetPath str args = append(args, "--project-id", testSuite.ProjectID) } - _, err := executeAzdCommand(ctx, testSuite, 2*time.Minute, args) + var err error + + if cli != nil { + _, err = executeAzdCommand(ctx, cli, testSuite, 2*time.Minute, args) + } else { + _, err = executeAzdCommandWithExec(ctx, testSuite, 2*time.Minute, args) + } + if err != nil { return err } @@ -168,7 +176,33 @@ func (e *InitError) Unwrap() error { } // executeAzdCommand executes an azd command and returns output and agent version if available -func executeAzdCommand(ctx context.Context, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { +func executeAzdCommand(ctx context.Context, cli *azdcli.CLI, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + Logf("Executing command: %s", strings.Join(args, " ")) + Logf("Working directory: %s", testSuite.AzdProjectDir) + + cli.WorkingDirectory = testSuite.AzdProjectDir + result, err := cli.RunCommandWithStdIn(ctx, "", args...) + output := result.Stdout + result.Stderr + + LogCommandOutput(strings.Join(args, " "), []byte(output)) + + if err != nil || result.ExitCode != 0 { + Logf("Command failed with error: %v, exit code: %d", err, result.ExitCode) + return "", &InitError{ + Message: output, + Err: fmt.Errorf("exit code %d: %w", result.ExitCode, err), + } + } + Logf("Command completed successfully") + + return output, nil +} + +// executeAzdCommand executes an azd command and returns output and agent version if available +func executeAzdCommandWithExec(ctx context.Context, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -222,10 +256,10 @@ func parseOutputForAgentVersion(output string) string { // ExecuteUpCommandForAgent executes the AZD up command with the given parameters // Returns the deployed agent version number if successful -func ExecuteUpCommandForAgent(ctx context.Context, testSuite *IntegrationTestSuite) (string, error) { +func ExecuteUpCommandForAgent(ctx context.Context, cli *azdcli.CLI, testSuite *IntegrationTestSuite) (string, error) { args := []string{"up", "--no-prompt"} - output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) + output, err := executeAzdCommand(ctx, cli, testSuite, 20*time.Minute, args) if err != nil { return "", err } @@ -237,7 +271,7 @@ func ExecuteUpCommandForAgent(ctx context.Context, testSuite *IntegrationTestSui // ExecuteDeployCommandForAgent executes the AZD deploy command with the given parameters // Returns the deployed agent version number if successful -func ExecuteDeployCommandForAgent(ctx context.Context, agentName string, testSuite *IntegrationTestSuite) (string, error) { +func ExecuteDeployCommandForAgent(ctx context.Context, cli *azdcli.CLI, agentName string, testSuite *IntegrationTestSuite) (string, error) { // Prepare command arguments args := []string{"deploy"} if agentName != "" { @@ -245,7 +279,7 @@ func ExecuteDeployCommandForAgent(ctx context.Context, agentName string, testSui } args = append(args, "--no-prompt") - output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) + output, err := executeAzdCommand(ctx, cli, testSuite, 20*time.Minute, args) if err != nil { return "", err } From 8428dc0b7872d9bcb4284e19657ae1123da442db Mon Sep 17 00:00:00 2001 From: trangevi Date: Tue, 6 Jan 2026 09:20:34 -0800 Subject: [PATCH 4/7] Fix merge Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index 8c8d44fcc0b..c4cf5a0b5b4 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -17,6 +17,8 @@ require ( github.com/google/uuid v1.6.0 github.com/mark3labs/mcp-go v0.41.1 github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 From ac95dd4f3c176bdaac74f5a4f6fb7021aef1adcc Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 8 Jan 2026 14:01:33 -0800 Subject: [PATCH 5/7] Revert "Some changes for test recording" This reverts commit f91db64d576dbcfb9f5a085038ba80eca7b61387. --- cli/azd/extensions/azure.ai.agents/go.mod | 5 +- cli/azd/extensions/azure.ai.agents/go.sum | 8 ---- .../deployTests/deploy_test.go | 11 +---- .../integrationTests/initTests/init_test.go | 5 +- .../testUtilities/test_suite.go | 10 ++-- .../testUtilities/test_utils.go | 48 +++---------------- 6 files changed, 15 insertions(+), 72 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index c4cf5a0b5b4..f236dc2e8b3 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -18,7 +18,6 @@ require ( github.com/mark3labs/mcp-go v0.41.1 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 - github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 @@ -28,9 +27,7 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v3 v3.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect @@ -79,6 +76,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -98,5 +96,4 @@ require ( golang.org/x/text v0.31.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251007200510-49b9836ed3ff // indirect google.golang.org/grpc v1.76.0 // indirect - gopkg.in/dnaeon/go-vcr.v3 v3.2.0 // indirect ) diff --git a/cli/azd/extensions/azure.ai.agents/go.sum b/cli/azd/extensions/azure.ai.agents/go.sum index 4858070ab8d..5045ad2c862 100644 --- a/cli/azd/extensions/azure.ai.agents/go.sum +++ b/cli/azd/extensions/azure.ai.agents/go.sum @@ -13,8 +13,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDo github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers v1.1.0 h1:fdAOz6TFldGDoEcRa975i5L5QvWU8ptut+SJAIfuWUY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers v1.1.0/go.mod h1:qV+BWew22CAalRTwJEAHs+aSLP49k/csNlspqhMIDRU= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v3 v3.1.0 h1:ilMZ576u8sm975EqV+AKEtD4u9TLwqEo2XY9csPXBRo= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v3 v3.1.0/go.mod h1:LGhzy+pg9AKr1Z7ZRyTC1qr1xNyVqLsqydvLdY+2iQk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= @@ -23,8 +21,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2/go.mod h1:jVRrRDLCOuif95HDYC23ADTMlvahB7tMdl519m9Iyjc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cognitiveservices/armcognitiveservices v1.8.0 h1:ZMGAqCZov8+7iFUPWKVcTaLgNXUeTlz20sIuWkQWNfg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cognitiveservices/armcognitiveservices v1.8.0/go.mod h1:BElPQ/GZtrdQ2i5uDZw3OKLE1we75W0AEWyeBR1TWQA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= @@ -33,8 +29,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= @@ -304,8 +298,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= -gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go index a3185c9be6a..87111c380b3 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/deployTests/deploy_test.go @@ -18,8 +18,6 @@ import ( "azureaiagent/test/integrationTests/testUtilities" - "github.com/azure/azure-dev/cli/azd/test/azdcli" - "github.com/azure/azure-dev/cli/azd/test/recording" "github.com/stretchr/testify/require" ) @@ -86,13 +84,8 @@ func TestDeployCommand_Integration(t *testing.T) { testUtilities.SetCurrentTestName(tt.name) testUtilities.Logf("Running test: %s", tt.name) - session := recording.Start(t) - - cli := azdcli.NewCLI(t, azdcli.WithSession(session)) - cli.WorkingDirectory = deployTestSuite.AzdProjectDir - // Execute init command - err := testUtilities.ExecuteInitCommandForAgent(context.Background(), cli, tt.manifestURL, "", deployTestSuite) + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, "", deployTestSuite) require.NoError(t, err) @@ -100,7 +93,7 @@ func TestDeployCommand_Integration(t *testing.T) { testUtilities.VerifyInitializedProject(t, deployTestSuite, "", tt.agentName) // Execute deploy command - agentVersion, err := testUtilities.ExecuteDeployCommandForAgent(context.Background(), cli, tt.agentName, deployTestSuite) + agentVersion, err := testUtilities.ExecuteDeployCommandForAgent(context.Background(), tt.agentName, deployTestSuite) if tt.wantErr { require.Error(t, err) testUtilities.Logf("Test completed (expected error)") diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go index 2786efaa915..bc1bf9bbe11 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/initTests/init_test.go @@ -12,7 +12,6 @@ import ( "azureaiagent/test/integrationTests/testUtilities" - "github.com/azure/azure-dev/cli/azd/test/azdcli" "github.com/stretchr/testify/require" ) @@ -105,10 +104,8 @@ func TestInitCommand_Integration(t *testing.T) { testUtilities.SetCurrentTestName(tt.name) testUtilities.Logf("Running test: %s", tt.name) - cli := azdcli.NewCLI(t) - // Execute init command - err := testUtilities.ExecuteInitCommandForAgent(context.Background(), cli, tt.manifestURL, tt.targetDir, testSuite) + err := testUtilities.ExecuteInitCommandForAgent(context.Background(), tt.manifestURL, tt.targetDir, testSuite) if tt.wantErr { require.Error(t, err) testUtilities.Logf("Test completed (expected error)") diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go index 51e01ad7ec9..f872bd99869 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go @@ -129,9 +129,7 @@ func (s *IntegrationTestSuite) initializeAzdProject() error { } // Initialize a calculator agent into the project so we have the model we need - if err := ExecuteInitCommandForAgent( - context.Background(), - nil, + if err := ExecuteInitCommandForAgent(context.Background(), "https://github.com/azure-ai-foundry/foundry-samples/blob/main/samples/python/hosted-agents/calculator-agent/agent.yaml", "", s); err != nil { return fmt.Errorf("failed to initialize calculator agent: %w", err) @@ -174,7 +172,7 @@ func (s *IntegrationTestSuite) runAzdInit(withTemplate bool) error { args = append(args, "-t", "Azure-Samples/azd-ai-starter-basic") } - _, err := executeAzdCommandWithExec(context.Background(), s, 5*time.Minute, args) + _, err := executeAzdCommand(context.Background(), s, 5*time.Minute, args) if err != nil { return fmt.Errorf("azd init command failed: %w", err) } @@ -188,7 +186,7 @@ func (s *IntegrationTestSuite) runAzdUp() error { args := []string{"up", "--no-prompt"} - _, err := executeAzdCommandWithExec(context.Background(), s, 10*time.Minute, args) + _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) if err != nil { return fmt.Errorf("azd up command failed: %w", err) } @@ -202,7 +200,7 @@ func (s *IntegrationTestSuite) runAzdDown() error { args := []string{"down", "--force", "--purge", "--no-prompt"} - _, err := executeAzdCommandWithExec(context.Background(), s, 10*time.Minute, args) + _, err := executeAzdCommand(context.Background(), s, 10*time.Minute, args) if err != nil { Logf("Warning: Azure cleanup failed: %v", err) } diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go index d813f6fff00..7bf6ce9ec1e 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/azure/azure-dev/cli/azd/test/azdcli" "github.com/stretchr/testify/require" ) @@ -64,7 +63,7 @@ func SetCurrentTestName(name string) { } // ExecuteInitCommandForAgent executes the AI agent init command with the given parameters -func ExecuteInitCommandForAgent(ctx context.Context, cli *azdcli.CLI, manifestURL, targetPath string, testSuite *IntegrationTestSuite) error { +func ExecuteInitCommandForAgent(ctx context.Context, manifestURL, targetPath string, testSuite *IntegrationTestSuite) error { // Prepare command arguments args := []string{"ai", "agent", "init", "--no-prompt"} @@ -81,14 +80,7 @@ func ExecuteInitCommandForAgent(ctx context.Context, cli *azdcli.CLI, manifestUR args = append(args, "--project-id", testSuite.ProjectID) } - var err error - - if cli != nil { - _, err = executeAzdCommand(ctx, cli, testSuite, 2*time.Minute, args) - } else { - _, err = executeAzdCommandWithExec(ctx, testSuite, 2*time.Minute, args) - } - + _, err := executeAzdCommand(ctx, testSuite, 2*time.Minute, args) if err != nil { return err } @@ -176,33 +168,7 @@ func (e *InitError) Unwrap() error { } // executeAzdCommand executes an azd command and returns output and agent version if available -func executeAzdCommand(ctx context.Context, cli *azdcli.CLI, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - Logf("Executing command: %s", strings.Join(args, " ")) - Logf("Working directory: %s", testSuite.AzdProjectDir) - - cli.WorkingDirectory = testSuite.AzdProjectDir - result, err := cli.RunCommandWithStdIn(ctx, "", args...) - output := result.Stdout + result.Stderr - - LogCommandOutput(strings.Join(args, " "), []byte(output)) - - if err != nil || result.ExitCode != 0 { - Logf("Command failed with error: %v, exit code: %d", err, result.ExitCode) - return "", &InitError{ - Message: output, - Err: fmt.Errorf("exit code %d: %w", result.ExitCode, err), - } - } - Logf("Command completed successfully") - - return output, nil -} - -// executeAzdCommand executes an azd command and returns output and agent version if available -func executeAzdCommandWithExec(ctx context.Context, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { +func executeAzdCommand(ctx context.Context, testSuite *IntegrationTestSuite, timeout time.Duration, args []string) (string, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -256,10 +222,10 @@ func parseOutputForAgentVersion(output string) string { // ExecuteUpCommandForAgent executes the AZD up command with the given parameters // Returns the deployed agent version number if successful -func ExecuteUpCommandForAgent(ctx context.Context, cli *azdcli.CLI, testSuite *IntegrationTestSuite) (string, error) { +func ExecuteUpCommandForAgent(ctx context.Context, testSuite *IntegrationTestSuite) (string, error) { args := []string{"up", "--no-prompt"} - output, err := executeAzdCommand(ctx, cli, testSuite, 20*time.Minute, args) + output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) if err != nil { return "", err } @@ -271,7 +237,7 @@ func ExecuteUpCommandForAgent(ctx context.Context, cli *azdcli.CLI, testSuite *I // ExecuteDeployCommandForAgent executes the AZD deploy command with the given parameters // Returns the deployed agent version number if successful -func ExecuteDeployCommandForAgent(ctx context.Context, cli *azdcli.CLI, agentName string, testSuite *IntegrationTestSuite) (string, error) { +func ExecuteDeployCommandForAgent(ctx context.Context, agentName string, testSuite *IntegrationTestSuite) (string, error) { // Prepare command arguments args := []string{"deploy"} if agentName != "" { @@ -279,7 +245,7 @@ func ExecuteDeployCommandForAgent(ctx context.Context, cli *azdcli.CLI, agentNam } args = append(args, "--no-prompt") - output, err := executeAzdCommand(ctx, cli, testSuite, 20*time.Minute, args) + output, err := executeAzdCommand(ctx, testSuite, 20*time.Minute, args) if err != nil { return "", err } From 3b8f593f222169881b4f3b57e90ec3893cca242c Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 8 Jan 2026 14:02:13 -0800 Subject: [PATCH 6/7] fix .mod file Signed-off-by: trangevi --- cli/azd/extensions/azure.ai.agents/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index f236dc2e8b3..fe00b76aa63 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -18,6 +18,7 @@ require ( github.com/mark3labs/mcp-go v0.41.1 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 @@ -76,7 +77,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect From d28e640cfc04840989c9e4fc5b0284671c450711 Mon Sep 17 00:00:00 2001 From: trangevi Date: Thu, 8 Jan 2026 15:51:02 -0800 Subject: [PATCH 7/7] Couple of PR comments Signed-off-by: trangevi --- .../test/integrationTests/testUtilities/test_suite.go | 4 +--- .../test/integrationTests/testUtilities/test_utils.go | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go index f872bd99869..8e95be0d19a 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_suite.go @@ -195,7 +195,7 @@ func (s *IntegrationTestSuite) runAzdUp() error { } // runAzdDown executes the azd down command to clean up Azure resources -func (s *IntegrationTestSuite) runAzdDown() error { +func (s *IntegrationTestSuite) runAzdDown() { Logf("Cleaning up Azure resources...") args := []string{"down", "--force", "--purge", "--no-prompt"} @@ -204,8 +204,6 @@ func (s *IntegrationTestSuite) runAzdDown() error { if err != nil { Logf("Warning: Azure cleanup failed: %v", err) } - - return nil // Always return nil to not block cleanup } // GetAzdEnvValue retrieves an environment variable value from the azd environment diff --git a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go index 7bf6ce9ec1e..07e45777d68 100644 --- a/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go +++ b/cli/azd/extensions/azure.ai.agents/test/integrationTests/testUtilities/test_utils.go @@ -214,6 +214,8 @@ func parseOutputForAgentVersion(output string) string { agentVersion = matches[1] Logf("Parsed agent version: %s", agentVersion) } else { + // Don't fail in the case we can't parse the agent version, just log a warning + // This allows tests to individually handle whether they require the version or not Logf("Warning: Could not parse agent version from output") }