diff --git a/.github/workflows/server-test.yaml b/.github/workflows/server-test.yaml index 620b8fb7..f65ae214 100644 --- a/.github/workflows/server-test.yaml +++ b/.github/workflows/server-test.yaml @@ -60,5 +60,5 @@ jobs: run: make test working-directory: server env: - # Prefer explicit images if passed from the caller; otherwise use the commit short sha - E2E_IMAGE_TAG: ${{ steps.vars.outputs.short_sha }} + E2E_CHROMIUM_HEADFUL_IMAGE: onkernel/chromium-headful:${{ steps.vars.outputs.short_sha }} + E2E_CHROMIUM_HEADLESS_IMAGE: onkernel/chromium-headless:${{ steps.vars.outputs.short_sha }} diff --git a/server/Makefile b/server/Makefile index 83e78c96..d9952048 100644 --- a/server/Makefile +++ b/server/Makefile @@ -35,7 +35,9 @@ dev: build $(RECORDING_DIR) test: go vet ./... - go test -v -race ./... + # Run tests sequentially (-p 1) to avoid port conflicts in e2e tests + # (all e2e tests bind to the same ports: 10001, 9222) + go test -v -race -p 1 ./... clean: @rm -rf $(BIN_DIR) diff --git a/server/e2e/e2e_chromium_restart_bench_test.go b/server/e2e/e2e_chromium_restart_bench_test.go index 1878490d..fa07eba9 100644 --- a/server/e2e/e2e_chromium_restart_bench_test.go +++ b/server/e2e/e2e_chromium_restart_bench_test.go @@ -316,7 +316,7 @@ func TestChromiumRestartTiming(t *testing.T) { defer cancel() t.Logf("Waiting for API...") - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready") + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready") t.Logf("Waiting for DevTools...") require.NoError(t, waitTCP(ctx, "127.0.0.1:9222"), "DevTools not ready") diff --git a/server/e2e/e2e_chromium_test.go b/server/e2e/e2e_chromium_test.go index 1c08f78c..2dd98def 100644 --- a/server/e2e/e2e_chromium_test.go +++ b/server/e2e/e2e_chromium_test.go @@ -112,7 +112,7 @@ func TestDisplayResolutionChange(t *testing.T) { defer cancel() logger.Info("[setup]", "action", "waiting for API", "url", apiBaseURL+"/spec.yaml") - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) client, err := apiClient() require.NoError(t, err, "failed to create API client: %v", err) @@ -212,7 +212,7 @@ func TestExtensionUploadAndActivation(t *testing.T) { ctx, cancel := context.WithTimeout(baseCtx, 3*time.Minute) defer cancel() - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) // Wait for DevTools _, err = waitDevtoolsWS(ctx) @@ -306,7 +306,7 @@ func TestScreenshotHeadless(t *testing.T) { ctx, cancel := context.WithTimeout(baseCtx, 2*time.Minute) defer cancel() - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) client, err := apiClient() require.NoError(t, err) @@ -357,7 +357,7 @@ func TestScreenshotHeadful(t *testing.T) { ctx, cancel := context.WithTimeout(baseCtx, 2*time.Minute) defer cancel() - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) client, err := apiClient() require.NoError(t, err) @@ -402,7 +402,7 @@ func TestInputEndpointsSmoke(t *testing.T) { ctx, cancel := context.WithTimeout(baseCtx, 2*time.Minute) defer cancel() - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) client, err := apiClient() require.NoError(t, err) @@ -445,17 +445,46 @@ func isPNG(data []byte) bool { return true } +// ContainerOptions configures container startup behavior +type ContainerOptions struct { + // HostAccess adds --add-host=host.docker.internal:host-gateway for tests + // that need to reach services on the host machine + HostAccess bool +} + func runContainer(ctx context.Context, image, name string, env map[string]string) (*exec.Cmd, <-chan error, error) { + return runContainerWithOptions(ctx, image, name, env, ContainerOptions{}) +} + +func runContainerWithOptions(ctx context.Context, image, name string, env map[string]string, opts ContainerOptions) (*exec.Cmd, <-chan error, error) { logger := logctx.FromContext(ctx) args := []string{ "run", "--name", name, "--privileged", "-p", "10001:10001", // API server - "-p", "9222:9222", // DevTools proxy - "--tmpfs", "/dev/shm:size=2g", + "-p", "9222:9222", // DevTools proxy + "--tmpfs", "/dev/shm:size=2g,mode=1777", } + + if opts.HostAccess { + args = append(args, "--add-host=host.docker.internal:host-gateway") + } + + // Ensure CHROMIUM_FLAGS includes --no-sandbox for CI environments where + // unprivileged user namespaces may be disabled (e.g., Ubuntu 24.04 GitHub Actions) + // Create a copy to avoid mutating the caller's map + envCopy := make(map[string]string) for k, v := range env { + envCopy[k] = v + } + if _, ok := envCopy["CHROMIUM_FLAGS"]; !ok { + envCopy["CHROMIUM_FLAGS"] = "--no-sandbox" + } else if !strings.Contains(envCopy["CHROMIUM_FLAGS"], "--no-sandbox") { + envCopy["CHROMIUM_FLAGS"] = envCopy["CHROMIUM_FLAGS"] + " --no-sandbox" + } + + for k, v := range envCopy { args = append(args, "-e", fmt.Sprintf("%s=%s", k, v)) } args = append(args, image) @@ -515,6 +544,73 @@ func stopContainer(ctx context.Context, name string) error { return nil } +// getContainerLogs retrieves the last N lines of container logs for debugging. +// Uses a fresh context with its own timeout to avoid issues when the parent context is cancelled. +func getContainerLogs(_ context.Context, name string, tailLines int) string { + // Use a fresh context with generous timeout - the parent context may be cancelled + logCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cmd := exec.CommandContext(logCtx, "docker", "logs", "--tail", fmt.Sprintf("%d", tailLines), name) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Sprintf("failed to get container logs: %v", err) + } + return string(output) +} + +// waitHTTPOrExitWithLogs waits for HTTP endpoint and captures container logs on failure. +// It also periodically logs container status during the wait for better visibility. +func waitHTTPOrExitWithLogs(ctx context.Context, url string, exitCh <-chan error, containerName string) error { + logger := logctx.FromContext(ctx) + + // Start a background goroutine to periodically show container status + // Use a separate stopCh that we close to signal the goroutine to stop, + // avoiding the race condition of sending to a potentially closed channel + stopCh := make(chan struct{}) + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-stopCh: + return + case <-ticker.C: + // Check if container is still running + checkCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + cmd := exec.CommandContext(checkCtx, "docker", "inspect", "--format", "{{.State.Status}} (pid={{.State.Pid}}, started={{.State.StartedAt}})", containerName) + out, err := cmd.Output() + cancel() + if err == nil { + logger.Info("[container-status]", "container", containerName, "status", strings.TrimSpace(string(out))) + } + // Also show last few log lines + recentLogs := getContainerLogs(ctx, containerName, 10) + if recentLogs != "" && !strings.Contains(recentLogs, "failed to get") { + logger.Info("[container-logs]", "recent_output", strings.TrimSpace(recentLogs)) + } + } + } + }() + + err := waitHTTPOrExit(ctx, url, exitCh) + + // Signal the status goroutine to stop and wait for it to finish + close(stopCh) + <-doneCh + + if err != nil { + // Capture container logs for debugging + logs := getContainerLogs(ctx, containerName, 100) + return fmt.Errorf("%w\n\nContainer logs (last 100 lines):\n%s", err, logs) + } + return nil +} + func waitHTTPOrExit(ctx context.Context, url string, exitCh <-chan error) error { client := &http.Client{Timeout: 5 * time.Second} ticker := time.NewTicker(500 * time.Millisecond) @@ -756,7 +852,7 @@ func TestCDPTargetCreation(t *testing.T) { defer cancel() logger.Info("[test]", "action", "waiting for API") - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready") + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready") // Wait for CDP endpoint to be ready (via the devtools proxy) logger.Info("[test]", "action", "waiting for CDP endpoint") @@ -830,7 +926,7 @@ func TestWebBotAuthInstallation(t *testing.T) { defer cancel() logger.Info("[setup]", "action", "waiting for API", "url", apiBaseURL+"/spec.yaml") - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) // Build mock web-bot-auth extension zip in-memory extDir := t.TempDir() diff --git a/server/e2e/e2e_playwright_test.go b/server/e2e/e2e_playwright_test.go index 76c3645a..866fbacc 100644 --- a/server/e2e/e2e_playwright_test.go +++ b/server/e2e/e2e_playwright_test.go @@ -36,7 +36,7 @@ func TestPlaywrightExecuteAPI(t *testing.T) { ctx, cancel := context.WithTimeout(baseCtx, 2*time.Minute) defer cancel() - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) client, err := apiClient() require.NoError(t, err) @@ -114,7 +114,7 @@ func TestPlaywrightDaemonRecovery(t *testing.T) { ctx, cancel := context.WithTimeout(baseCtx, 3*time.Minute) defer cancel() - require.NoError(t, waitHTTPOrExit(ctx, apiBaseURL+"/spec.yaml", exitCh), "api not ready: %v", err) + require.NoError(t, waitHTTPOrExitWithLogs(ctx, apiBaseURL+"/spec.yaml", exitCh, name), "api not ready: %v", err) client, err := apiClient() require.NoError(t, err)