diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 659f1b9b48..85f8eb0484 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -86,5 +86,7 @@ jobs: testrun: ${{fromJson(needs.e2e-testruns.outputs.testrun)}} steps: - uses: actions/checkout@v4 - - name: Run e2e + - name: Run e2e (with GCM) + env: + GCM_SECRET: ${{ secrets.GCM_SECRET }} run: TEST_RUN=${{matrix.testrun}} make e2e diff --git a/Makefile b/Makefile index 2828ed1c2a..c143de48aa 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,14 @@ TEST_ARGS=-project-id=$(PROJECT_ID) -location=$(GMP_LOCATION) -cluster=$(GMP_CLU API_DIR=pkg/operator/apis LOCAL_CREDENTIALS=/tmp/gcm-editor.json -# If credentials are provided, ensure we mount them during e2e test. -ifneq ($(GOOGLE_APPLICATION_CREDENTIALS),) -E2E_DOCKER_ARGS := --env GOOGLE_APPLICATION_CREDENTIALS="$(LOCAL_CREDENTIALS)" -v $(GOOGLE_APPLICATION_CREDENTIALS):$(LOCAL_CREDENTIALS) +ifneq ($(GCM_SECRET),) + # If GCM_SECRET env-var is provided ensure we add it to e2e tests. + # Don't pass the explicit value here, see: + # https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-secrets + E2E_DOCKER_ARGS := --env GCM_SECRET +else ifneq ($(GOOGLE_APPLICATION_CREDENTIALS),) + # If credentials are provided, ensure we mount them during e2e test. + E2E_DOCKER_ARGS := --env GOOGLE_APPLICATION_CREDENTIALS="$(LOCAL_CREDENTIALS)" -v $(GOOGLE_APPLICATION_CREDENTIALS):$(LOCAL_CREDENTIALS) endif ifeq ($(KIND_PERSIST), 1) diff --git a/e2e/collector_test.go b/e2e/collector_test.go index a1564838b5..5ee52c9c24 100644 --- a/e2e/collector_test.go +++ b/e2e/collector_test.go @@ -22,7 +22,6 @@ import ( "testing" "time" - gcm "cloud.google.com/go/monitoring/apiv3/v2" gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" "github.com/GoogleCloudPlatform/prometheus-engine/e2e/kube" monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1" @@ -458,7 +457,7 @@ func testValidateCollectorUpMetrics(ctx context.Context, kubeClient client.Clien t.Log("checking for metrics in Cloud Monitoring") // Wait for metric data to show up in Cloud Monitoring. - metricClient, err := gcm.NewMetricClient(ctx) + metricClient, err := newMetricClient(ctx) if err != nil { t.Fatalf("create metric client: %s", err) } @@ -537,7 +536,7 @@ func testCollectorScrapeKubelet(ctx context.Context, kubeClient client.Client) f t.Log("checking for metrics in Cloud Monitoring") // Wait for metric data to show up in Cloud Monitoring. - metricClient, err := gcm.NewMetricClient(ctx) + metricClient, err := newMetricClient(ctx) if err != nil { t.Fatalf("create GCM metric client: %s", err) } diff --git a/e2e/deploy/deploy.go b/e2e/deploy/deploy.go index 33daeb4406..ae86d86a05 100644 --- a/e2e/deploy/deploy.go +++ b/e2e/deploy/deploy.go @@ -182,17 +182,22 @@ func normalizeDeployments(opts *deployOptions, obj *appsv1.Deployment) (client.O return obj, nil } -func CreateGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string, serviceAccount []byte) error { +const ( + GCMSecretName = "user-gcp-service-account" + GCMSecretKey = "key.json" +) + +func CreateGCMSecret(ctx context.Context, kubeClient client.Client, serviceAccount []byte) error { if err := kubeClient.Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "user-gcp-service-account", - Namespace: namespace, + Namespace: operator.DefaultOperatorNamespace, }, Data: map[string][]byte{ "key.json": serviceAccount, }, }); err != nil { - return fmt.Errorf("create GCP service account secret: %w", err) + return fmt.Errorf("create GCM service account secret: %w", err) } return nil } diff --git a/e2e/main_test.go b/e2e/main_test.go index 1e602f9234..406953fcd2 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -20,15 +20,20 @@ package e2e import ( "context" + "errors" "flag" "fmt" "os" "testing" "time" + gcm "cloud.google.com/go/monitoring/apiv3/v2" + monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1" "go.uber.org/zap/zapcore" - metav1 "k8s.io/api/core/v1" + "google.golang.org/api/option" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" @@ -47,6 +52,7 @@ const ( // Arbitrary amount of time to let tests exit cleanly before the main process // terminates. timeoutGracePeriod = 10 * time.Second + gcmSecretEnv = "GCM_SECRET" ) var ( @@ -86,7 +92,7 @@ func setupCluster(ctx context.Context, t testing.TB) (client.Client, *rest.Confi return nil, nil, err } - if err := createResources(ctx, kubeClient); err != nil { + if err := deploy.CreateResources(ctx, kubeClient, deploy.WithMeta(projectID, cluster, location), deploy.WithDisableGCM(skipGCM)); err != nil { return nil, nil, err } @@ -99,9 +105,83 @@ func setupCluster(ctx context.Context, t testing.TB) (client.Client, *rest.Confi return nil, nil, err } t.Log(">>> operator started successfully") + + if explicitCredentialsConfigured() { + t.Log(">>> setup explicit credentials") + if err := setupExplicitCredentials(ctx, kubeClient); err != nil { + return nil, nil, err + } + } return kubeClient, restConfig, nil } +// Setup explicit credentials for GCM_SECRET or explicit gcpServiceAccount flag. +// Otherwise, GOOGLE_APPLICATION_CREDENTIALS will use default flow. +func explicitCredentialsConfigured() bool { + return !skipGCM && (gcpServiceAccount != "" || os.Getenv(gcmSecretEnv) != "") +} + +func getExplicitGCMSAJSON() ([]byte, error) { + if gcpServiceAccount != "" { + b, err := os.ReadFile(gcpServiceAccount) + if err != nil { + return b, fmt.Errorf("read service account file %q: %w", gcpServiceAccount, err) + } + return b, nil + } + if gcmSA := os.Getenv(gcmSecretEnv); gcmSA != "" { + return []byte(gcmSA), nil + } + return nil, errors.New("gcp-service-account flag or GCM_SECRET environment variable not set") +} + +// newMetricClient returns GCM client. Our e2e tests supports both GOOGLE_APPLICATION_CREDENTIALS (file) +// and GCM_SECRET (content, required with CI use). +func newMetricClient(ctx context.Context) (*gcm.MetricClient, error) { + if !explicitCredentialsConfigured() { + return gcm.NewMetricClient(ctx) + } + gcmSA, err := getExplicitGCMSAJSON() + if err != nil { + return nil, err + } + return gcm.NewMetricClient(ctx, option.WithCredentialsJSON(gcmSA)) +} + +func setupExplicitCredentials(ctx context.Context, kubeClient client.Client) error { + gcmSA, err := getExplicitGCMSAJSON() + if err != nil { + return err + } + // Deploy gmp-public secret. + if err := deploy.CreateGCMSecret(ctx, kubeClient, gcmSA); err != nil { + return err + } + // Select it in OperatorConfig. + config := monitoringv1.OperatorConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: operator.NameOperatorConfig, + Namespace: operator.DefaultPublicNamespace, + }, + } + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(&config), &config); err != nil { + return fmt.Errorf("get operatorconfig: %w", err) + } + config.Collection.Credentials = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: deploy.GCMSecretName}, + Key: deploy.GCMSecretKey, + } + config.Rules.Credentials = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: deploy.GCMSecretName}, + Key: deploy.GCMSecretKey, + } + // Update OperatorConfig. + if err := kubeClient.Update(ctx, &config); err != nil { + return fmt.Errorf("update operatorconfig: %w", err) + } + return nil +} + func setRESTConfigDefaults(restConfig *rest.Config) error { // https://github.com/kubernetes/client-go/issues/657 // https://github.com/kubernetes/client-go/issues/1159 @@ -153,26 +233,6 @@ func newScheme() (*runtime.Scheme, error) { return scheme, nil } -func createResources(ctx context.Context, kubeClient client.Client) error { - if err := deploy.CreateResources(ctx, kubeClient, deploy.WithMeta(projectID, cluster, location), deploy.WithDisableGCM(skipGCM)); err != nil { - return err - } - - if gcpServiceAccount == "" { - gcpServiceAccount, _ = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS") - } - if gcpServiceAccount != "" { - b, err := os.ReadFile(gcpServiceAccount) - if err != nil { - return fmt.Errorf("read service account file %q: %w", gcpServiceAccount, err) - } - if err := deploy.CreateGCPSecretResources(context.Background(), kubeClient, metav1.NamespaceDefault, b); err != nil { - return err - } - } - return nil -} - // contextWithDeadline returns a context that will timeout before t.Deadline(). // See: https://github.com/golang/go/issues/36532 func contextWithDeadline(t *testing.T) context.Context { diff --git a/e2e/ruler_test.go b/e2e/ruler_test.go index f10a80ee1b..664cde5753 100644 --- a/e2e/ruler_test.go +++ b/e2e/ruler_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - gcm "cloud.google.com/go/monitoring/apiv3/v2" gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" "github.com/GoogleCloudPlatform/prometheus-engine/e2e/kube" "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator" @@ -648,7 +647,7 @@ func testValidateRuleEvaluationMetrics(ctx context.Context) func(*testing.T) { t.Log("checking for metrics in Cloud Monitoring") // Wait for metric data to show up in Cloud Monitoring. - metricClient, err := gcm.NewMetricClient(ctx) + metricClient, err := newMetricClient(ctx) if err != nil { t.Fatalf("create metric client: %s", err) } diff --git a/hack/kind-test.sh b/hack/kind-test.sh index 6a4b46ced4..20ba170fc4 100755 --- a/hack/kind-test.sh +++ b/hack/kind-test.sh @@ -34,8 +34,14 @@ KIND_CLUSTER=${KIND_CLUSTER}-${KIND_CLUSTER_HASH} KUBECTL="kubectl --context kind-${KIND_CLUSTER}" # Ensure a unique label on any test data sent to GCM. GMP_CLUSTER=$TAG_NAME -if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then +if [[ -n "${GCM_SECRET:-}" ]]; then + echo ">>> using GCM_SECRET credentials; running with GCM validation with explicit credentials" + PROJECT_ID=$(echo "${GCM_SECRET}" | jq -r '.project_id') + export GOOGLE_APPLICATION_CREDENTIALS="" +elif [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then + echo ">>> using GOOGLE_APPLICATION_CREDENTIALS credentials; running with GCM validation" PROJECT_ID=$(jq -r '.project_id' "${GOOGLE_APPLICATION_CREDENTIALS}") + export GCM_EXPORT="" else echo ">>> no credentials specified. running without GCM validation" TEST_ARGS="${TEST_ARGS} -skip-gcm" @@ -123,6 +129,7 @@ connect_registry echo ">>> executing gmp e2e tests: ${GO_TEST}" # Note: a test failure here should exit non-zero and leave the cluster running # for debugging. + go test -v -timeout 10m "${REPO_ROOT}/e2e" -run "${GO_TEST:-.}" -args ${TEST_ARGS} # Delete cluster if it's not set to clean up post-test.