From 6d48169365c37ee4d1e5ffb2b93d59e973285086 Mon Sep 17 00:00:00 2001 From: bwplotka Date: Mon, 1 Sep 2025 12:19:34 +0100 Subject: [PATCH] feat: enable against GCM tests for e2e We plan to add more tests around this path, so checking if it's safe to enable or we need to do bit more work to make it per CI. We already have secret for other GCM e2e tests. Signed-off-by: bwplotka --- .github/workflows/presubmit.yml | 4 +- Makefile | 11 +++- e2e/collector_test.go | 5 +- e2e/deploy/deploy.go | 11 +++- e2e/main_test.go | 104 +++++++++++++++++++++++++------- e2e/ruler_test.go | 3 +- hack/kind-test.sh | 9 ++- 7 files changed, 112 insertions(+), 35 deletions(-) diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 659f1b9b48..85f8eb0484 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -86,5 +86,7 @@ jobs: testrun: ${{fromJson(needs.e2e-testruns.outputs.testrun)}} steps: - uses: actions/checkout@v4 - - name: Run e2e + - name: Run e2e (with GCM) + env: + GCM_SECRET: ${{ secrets.GCM_SECRET }} run: TEST_RUN=${{matrix.testrun}} make e2e diff --git a/Makefile b/Makefile index 2828ed1c2a..c143de48aa 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,14 @@ TEST_ARGS=-project-id=$(PROJECT_ID) -location=$(GMP_LOCATION) -cluster=$(GMP_CLU API_DIR=pkg/operator/apis LOCAL_CREDENTIALS=/tmp/gcm-editor.json -# If credentials are provided, ensure we mount them during e2e test. -ifneq ($(GOOGLE_APPLICATION_CREDENTIALS),) -E2E_DOCKER_ARGS := --env GOOGLE_APPLICATION_CREDENTIALS="$(LOCAL_CREDENTIALS)" -v $(GOOGLE_APPLICATION_CREDENTIALS):$(LOCAL_CREDENTIALS) +ifneq ($(GCM_SECRET),) + # If GCM_SECRET env-var is provided ensure we add it to e2e tests. + # Don't pass the explicit value here, see: + # https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-secrets + E2E_DOCKER_ARGS := --env GCM_SECRET +else ifneq ($(GOOGLE_APPLICATION_CREDENTIALS),) + # If credentials are provided, ensure we mount them during e2e test. + E2E_DOCKER_ARGS := --env GOOGLE_APPLICATION_CREDENTIALS="$(LOCAL_CREDENTIALS)" -v $(GOOGLE_APPLICATION_CREDENTIALS):$(LOCAL_CREDENTIALS) endif ifeq ($(KIND_PERSIST), 1) diff --git a/e2e/collector_test.go b/e2e/collector_test.go index a1564838b5..5ee52c9c24 100644 --- a/e2e/collector_test.go +++ b/e2e/collector_test.go @@ -22,7 +22,6 @@ import ( "testing" "time" - gcm "cloud.google.com/go/monitoring/apiv3/v2" gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" "github.com/GoogleCloudPlatform/prometheus-engine/e2e/kube" monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1" @@ -458,7 +457,7 @@ func testValidateCollectorUpMetrics(ctx context.Context, kubeClient client.Clien t.Log("checking for metrics in Cloud Monitoring") // Wait for metric data to show up in Cloud Monitoring. - metricClient, err := gcm.NewMetricClient(ctx) + metricClient, err := newMetricClient(ctx) if err != nil { t.Fatalf("create metric client: %s", err) } @@ -537,7 +536,7 @@ func testCollectorScrapeKubelet(ctx context.Context, kubeClient client.Client) f t.Log("checking for metrics in Cloud Monitoring") // Wait for metric data to show up in Cloud Monitoring. - metricClient, err := gcm.NewMetricClient(ctx) + metricClient, err := newMetricClient(ctx) if err != nil { t.Fatalf("create GCM metric client: %s", err) } diff --git a/e2e/deploy/deploy.go b/e2e/deploy/deploy.go index 33daeb4406..ae86d86a05 100644 --- a/e2e/deploy/deploy.go +++ b/e2e/deploy/deploy.go @@ -182,17 +182,22 @@ func normalizeDeployments(opts *deployOptions, obj *appsv1.Deployment) (client.O return obj, nil } -func CreateGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string, serviceAccount []byte) error { +const ( + GCMSecretName = "user-gcp-service-account" + GCMSecretKey = "key.json" +) + +func CreateGCMSecret(ctx context.Context, kubeClient client.Client, serviceAccount []byte) error { if err := kubeClient.Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "user-gcp-service-account", - Namespace: namespace, + Namespace: operator.DefaultOperatorNamespace, }, Data: map[string][]byte{ "key.json": serviceAccount, }, }); err != nil { - return fmt.Errorf("create GCP service account secret: %w", err) + return fmt.Errorf("create GCM service account secret: %w", err) } return nil } diff --git a/e2e/main_test.go b/e2e/main_test.go index 1e602f9234..406953fcd2 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -20,15 +20,20 @@ package e2e import ( "context" + "errors" "flag" "fmt" "os" "testing" "time" + gcm "cloud.google.com/go/monitoring/apiv3/v2" + monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1" "go.uber.org/zap/zapcore" - metav1 "k8s.io/api/core/v1" + "google.golang.org/api/option" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" @@ -47,6 +52,7 @@ const ( // Arbitrary amount of time to let tests exit cleanly before the main process // terminates. timeoutGracePeriod = 10 * time.Second + gcmSecretEnv = "GCM_SECRET" ) var ( @@ -86,7 +92,7 @@ func setupCluster(ctx context.Context, t testing.TB) (client.Client, *rest.Confi return nil, nil, err } - if err := createResources(ctx, kubeClient); err != nil { + if err := deploy.CreateResources(ctx, kubeClient, deploy.WithMeta(projectID, cluster, location), deploy.WithDisableGCM(skipGCM)); err != nil { return nil, nil, err } @@ -99,9 +105,83 @@ func setupCluster(ctx context.Context, t testing.TB) (client.Client, *rest.Confi return nil, nil, err } t.Log(">>> operator started successfully") + + if explicitCredentialsConfigured() { + t.Log(">>> setup explicit credentials") + if err := setupExplicitCredentials(ctx, kubeClient); err != nil { + return nil, nil, err + } + } return kubeClient, restConfig, nil } +// Setup explicit credentials for GCM_SECRET or explicit gcpServiceAccount flag. +// Otherwise, GOOGLE_APPLICATION_CREDENTIALS will use default flow. +func explicitCredentialsConfigured() bool { + return !skipGCM && (gcpServiceAccount != "" || os.Getenv(gcmSecretEnv) != "") +} + +func getExplicitGCMSAJSON() ([]byte, error) { + if gcpServiceAccount != "" { + b, err := os.ReadFile(gcpServiceAccount) + if err != nil { + return b, fmt.Errorf("read service account file %q: %w", gcpServiceAccount, err) + } + return b, nil + } + if gcmSA := os.Getenv(gcmSecretEnv); gcmSA != "" { + return []byte(gcmSA), nil + } + return nil, errors.New("gcp-service-account flag or GCM_SECRET environment variable not set") +} + +// newMetricClient returns GCM client. Our e2e tests supports both GOOGLE_APPLICATION_CREDENTIALS (file) +// and GCM_SECRET (content, required with CI use). +func newMetricClient(ctx context.Context) (*gcm.MetricClient, error) { + if !explicitCredentialsConfigured() { + return gcm.NewMetricClient(ctx) + } + gcmSA, err := getExplicitGCMSAJSON() + if err != nil { + return nil, err + } + return gcm.NewMetricClient(ctx, option.WithCredentialsJSON(gcmSA)) +} + +func setupExplicitCredentials(ctx context.Context, kubeClient client.Client) error { + gcmSA, err := getExplicitGCMSAJSON() + if err != nil { + return err + } + // Deploy gmp-public secret. + if err := deploy.CreateGCMSecret(ctx, kubeClient, gcmSA); err != nil { + return err + } + // Select it in OperatorConfig. + config := monitoringv1.OperatorConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: operator.NameOperatorConfig, + Namespace: operator.DefaultPublicNamespace, + }, + } + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(&config), &config); err != nil { + return fmt.Errorf("get operatorconfig: %w", err) + } + config.Collection.Credentials = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: deploy.GCMSecretName}, + Key: deploy.GCMSecretKey, + } + config.Rules.Credentials = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: deploy.GCMSecretName}, + Key: deploy.GCMSecretKey, + } + // Update OperatorConfig. + if err := kubeClient.Update(ctx, &config); err != nil { + return fmt.Errorf("update operatorconfig: %w", err) + } + return nil +} + func setRESTConfigDefaults(restConfig *rest.Config) error { // https://github.com/kubernetes/client-go/issues/657 // https://github.com/kubernetes/client-go/issues/1159 @@ -153,26 +233,6 @@ func newScheme() (*runtime.Scheme, error) { return scheme, nil } -func createResources(ctx context.Context, kubeClient client.Client) error { - if err := deploy.CreateResources(ctx, kubeClient, deploy.WithMeta(projectID, cluster, location), deploy.WithDisableGCM(skipGCM)); err != nil { - return err - } - - if gcpServiceAccount == "" { - gcpServiceAccount, _ = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS") - } - if gcpServiceAccount != "" { - b, err := os.ReadFile(gcpServiceAccount) - if err != nil { - return fmt.Errorf("read service account file %q: %w", gcpServiceAccount, err) - } - if err := deploy.CreateGCPSecretResources(context.Background(), kubeClient, metav1.NamespaceDefault, b); err != nil { - return err - } - } - return nil -} - // contextWithDeadline returns a context that will timeout before t.Deadline(). // See: https://github.com/golang/go/issues/36532 func contextWithDeadline(t *testing.T) context.Context { diff --git a/e2e/ruler_test.go b/e2e/ruler_test.go index f10a80ee1b..664cde5753 100644 --- a/e2e/ruler_test.go +++ b/e2e/ruler_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - gcm "cloud.google.com/go/monitoring/apiv3/v2" gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" "github.com/GoogleCloudPlatform/prometheus-engine/e2e/kube" "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator" @@ -648,7 +647,7 @@ func testValidateRuleEvaluationMetrics(ctx context.Context) func(*testing.T) { t.Log("checking for metrics in Cloud Monitoring") // Wait for metric data to show up in Cloud Monitoring. - metricClient, err := gcm.NewMetricClient(ctx) + metricClient, err := newMetricClient(ctx) if err != nil { t.Fatalf("create metric client: %s", err) } diff --git a/hack/kind-test.sh b/hack/kind-test.sh index 6a4b46ced4..20ba170fc4 100755 --- a/hack/kind-test.sh +++ b/hack/kind-test.sh @@ -34,8 +34,14 @@ KIND_CLUSTER=${KIND_CLUSTER}-${KIND_CLUSTER_HASH} KUBECTL="kubectl --context kind-${KIND_CLUSTER}" # Ensure a unique label on any test data sent to GCM. GMP_CLUSTER=$TAG_NAME -if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then +if [[ -n "${GCM_SECRET:-}" ]]; then + echo ">>> using GCM_SECRET credentials; running with GCM validation with explicit credentials" + PROJECT_ID=$(echo "${GCM_SECRET}" | jq -r '.project_id') + export GOOGLE_APPLICATION_CREDENTIALS="" +elif [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then + echo ">>> using GOOGLE_APPLICATION_CREDENTIALS credentials; running with GCM validation" PROJECT_ID=$(jq -r '.project_id' "${GOOGLE_APPLICATION_CREDENTIALS}") + export GCM_EXPORT="" else echo ">>> no credentials specified. running without GCM validation" TEST_ARGS="${TEST_ARGS} -skip-gcm" @@ -123,6 +129,7 @@ connect_registry echo ">>> executing gmp e2e tests: ${GO_TEST}" # Note: a test failure here should exit non-zero and leave the cluster running # for debugging. + go test -v -timeout 10m "${REPO_ROOT}/e2e" -run "${GO_TEST:-.}" -args ${TEST_ARGS} # Delete cluster if it's not set to clean up post-test.