Skip to content

release/0.17[BACKPORT]: feat: enable against GCM tests for e2e #1848

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bwplotka wants to merge 1 commit into
base: release/0.17
Choose a base branch
from
+112 −35
Open

release/0.17[BACKPORT]: feat: enable against GCM tests for e2e #1848

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/presubmit.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,5 +86,7 @@ jobs:
testrun: ${{fromJson(needs.e2e-testruns.outputs.testrun)}}
steps:
- uses: actions/checkout@v4
- name: Run e2e
- name: Run e2e (with GCM)
env:
GCM_SECRET: ${{ secrets.GCM_SECRET }}
run: TEST_RUN=${{matrix.testrun}} make e2e
11 changes: 8 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,14 @@ TEST_ARGS=-project-id=$(PROJECT_ID) -location=$(GMP_LOCATION) -cluster=$(GMP_CLU

API_DIR=pkg/operator/apis
LOCAL_CREDENTIALS=/tmp/gcm-editor.json
# If credentials are provided, ensure we mount them during e2e test.
ifneq ($(GOOGLE_APPLICATION_CREDENTIALS),)
E2E_DOCKER_ARGS := --env GOOGLE_APPLICATION_CREDENTIALS="$(LOCAL_CREDENTIALS)" -v $(GOOGLE_APPLICATION_CREDENTIALS):$(LOCAL_CREDENTIALS)
ifneq ($(GCM_SECRET),)
# If GCM_SECRET env-var is provided ensure we add it to e2e tests.
# Don't pass the explicit value here, see:
# https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-secrets
E2E_DOCKER_ARGS := --env GCM_SECRET
else ifneq ($(GOOGLE_APPLICATION_CREDENTIALS),)
# If credentials are provided, ensure we mount them during e2e test.
E2E_DOCKER_ARGS := --env GOOGLE_APPLICATION_CREDENTIALS="$(LOCAL_CREDENTIALS)" -v $(GOOGLE_APPLICATION_CREDENTIALS):$(LOCAL_CREDENTIALS)
endif

ifeq ($(KIND_PERSIST), 1)
Expand Down
5 changes: 2 additions & 3 deletions e2e/collector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"testing"
"time"

gcm "cloud.google.com/go/monitoring/apiv3/v2"
gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
"github.com/GoogleCloudPlatform/prometheus-engine/e2e/kube"
monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1"
Expand Down Expand Up @@ -458,7 +457,7 @@ func testValidateCollectorUpMetrics(ctx context.Context, kubeClient client.Clien
t.Log("checking for metrics in Cloud Monitoring")

// Wait for metric data to show up in Cloud Monitoring.
metricClient, err := gcm.NewMetricClient(ctx)
metricClient, err := newMetricClient(ctx)
if err != nil {
t.Fatalf("create metric client: %s", err)
}
Expand Down Expand Up @@ -537,7 +536,7 @@ func testCollectorScrapeKubelet(ctx context.Context, kubeClient client.Client) f
t.Log("checking for metrics in Cloud Monitoring")

// Wait for metric data to show up in Cloud Monitoring.
metricClient, err := gcm.NewMetricClient(ctx)
metricClient, err := newMetricClient(ctx)
if err != nil {
t.Fatalf("create GCM metric client: %s", err)
}
Expand Down
11 changes: 8 additions & 3 deletions e2e/deploy/deploy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,17 +182,22 @@ func normalizeDeployments(opts *deployOptions, obj *appsv1.Deployment) (client.O
return obj, nil
}

func CreateGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string, serviceAccount []byte) error {
const (
GCMSecretName = "user-gcp-service-account"
GCMSecretKey = "key.json"
)

func CreateGCMSecret(ctx context.Context, kubeClient client.Client, serviceAccount []byte) error {
if err := kubeClient.Create(ctx, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "user-gcp-service-account",
Namespace: namespace,
Namespace: operator.DefaultOperatorNamespace,
},
Data: map[string][]byte{
"key.json": serviceAccount,
},
}); err != nil {
return fmt.Errorf("create GCP service account secret: %w", err)
return fmt.Errorf("create GCM service account secret: %w", err)
}
return nil
}
104 changes: 82 additions & 22 deletions e2e/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,20 @@ package e2e

import (
"context"
"errors"
"flag"
"fmt"
"os"
"testing"
"time"

gcm "cloud.google.com/go/monitoring/apiv3/v2"
monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1"
"go.uber.org/zap/zapcore"
metav1 "k8s.io/api/core/v1"
"google.golang.org/api/option"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes/scheme"
Expand All @@ -47,6 +52,7 @@ const (
// Arbitrary amount of time to let tests exit cleanly before the main process
// terminates.
timeoutGracePeriod = 10 * time.Second
gcmSecretEnv = "GCM_SECRET"
)

var (
Expand Down Expand Up @@ -86,7 +92,7 @@ func setupCluster(ctx context.Context, t testing.TB) (client.Client, *rest.Confi
return nil, nil, err
}

if err := createResources(ctx, kubeClient); err != nil {
if err := deploy.CreateResources(ctx, kubeClient, deploy.WithMeta(projectID, cluster, location), deploy.WithDisableGCM(skipGCM)); err != nil {
return nil, nil, err
}

Expand All @@ -99,9 +105,83 @@ func setupCluster(ctx context.Context, t testing.TB) (client.Client, *rest.Confi
return nil, nil, err
}
t.Log(">>> operator started successfully")

if explicitCredentialsConfigured() {
t.Log(">>> setup explicit credentials")
if err := setupExplicitCredentials(ctx, kubeClient); err != nil {
return nil, nil, err
}
}
return kubeClient, restConfig, nil
}

// Setup explicit credentials for GCM_SECRET or explicit gcpServiceAccount flag.
// Otherwise, GOOGLE_APPLICATION_CREDENTIALS will use default flow.
func explicitCredentialsConfigured() bool {
return !skipGCM && (gcpServiceAccount != "" || os.Getenv(gcmSecretEnv) != "")
}

func getExplicitGCMSAJSON() ([]byte, error) {
if gcpServiceAccount != "" {
b, err := os.ReadFile(gcpServiceAccount)
if err != nil {
return b, fmt.Errorf("read service account file %q: %w", gcpServiceAccount, err)
}
return b, nil
}
if gcmSA := os.Getenv(gcmSecretEnv); gcmSA != "" {
return []byte(gcmSA), nil
}
return nil, errors.New("gcp-service-account flag or GCM_SECRET environment variable not set")
}

// newMetricClient returns GCM client. Our e2e tests supports both GOOGLE_APPLICATION_CREDENTIALS (file)
// and GCM_SECRET (content, required with CI use).
func newMetricClient(ctx context.Context) (*gcm.MetricClient, error) {
if !explicitCredentialsConfigured() {
return gcm.NewMetricClient(ctx)
}
gcmSA, err := getExplicitGCMSAJSON()
if err != nil {
return nil, err
}
return gcm.NewMetricClient(ctx, option.WithCredentialsJSON(gcmSA))
}

func setupExplicitCredentials(ctx context.Context, kubeClient client.Client) error {
gcmSA, err := getExplicitGCMSAJSON()
if err != nil {
return err
}
// Deploy gmp-public secret.
if err := deploy.CreateGCMSecret(ctx, kubeClient, gcmSA); err != nil {
return err
}
// Select it in OperatorConfig.
config := monitoringv1.OperatorConfig{
ObjectMeta: metav1.ObjectMeta{
Name: operator.NameOperatorConfig,
Namespace: operator.DefaultPublicNamespace,
},
}
if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(&config), &config); err != nil {
return fmt.Errorf("get operatorconfig: %w", err)
}
config.Collection.Credentials = &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: deploy.GCMSecretName},
Key: deploy.GCMSecretKey,
}
config.Rules.Credentials = &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: deploy.GCMSecretName},
Key: deploy.GCMSecretKey,
}
// Update OperatorConfig.
if err := kubeClient.Update(ctx, &config); err != nil {
return fmt.Errorf("update operatorconfig: %w", err)
}
return nil
}

func setRESTConfigDefaults(restConfig *rest.Config) error {
// https://github.com/kubernetes/client-go/issues/657
// https://github.com/kubernetes/client-go/issues/1159
Expand Down Expand Up @@ -153,26 +233,6 @@ func newScheme() (*runtime.Scheme, error) {
return scheme, nil
}

func createResources(ctx context.Context, kubeClient client.Client) error {
if err := deploy.CreateResources(ctx, kubeClient, deploy.WithMeta(projectID, cluster, location), deploy.WithDisableGCM(skipGCM)); err != nil {
return err
}

if gcpServiceAccount == "" {
gcpServiceAccount, _ = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
}
if gcpServiceAccount != "" {
b, err := os.ReadFile(gcpServiceAccount)
if err != nil {
return fmt.Errorf("read service account file %q: %w", gcpServiceAccount, err)
}
if err := deploy.CreateGCPSecretResources(context.Background(), kubeClient, metav1.NamespaceDefault, b); err != nil {
return err
}
}
return nil
}

// contextWithDeadline returns a context that will timeout before t.Deadline().
// See: https://github.com/golang/go/issues/36532
func contextWithDeadline(t *testing.T) context.Context {
Expand Down
3 changes: 1 addition & 2 deletions e2e/ruler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"testing"
"time"

gcm "cloud.google.com/go/monitoring/apiv3/v2"
gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
"github.com/GoogleCloudPlatform/prometheus-engine/e2e/kube"
"github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator"
Expand Down Expand Up @@ -648,7 +647,7 @@ func testValidateRuleEvaluationMetrics(ctx context.Context) func(*testing.T) {
t.Log("checking for metrics in Cloud Monitoring")

// Wait for metric data to show up in Cloud Monitoring.
metricClient, err := gcm.NewMetricClient(ctx)
metricClient, err := newMetricClient(ctx)
if err != nil {
t.Fatalf("create metric client: %s", err)
}
Expand Down
9 changes: 8 additions & 1 deletion hack/kind-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,14 @@ KIND_CLUSTER=${KIND_CLUSTER}-${KIND_CLUSTER_HASH}
KUBECTL="kubectl --context kind-${KIND_CLUSTER}"
# Ensure a unique label on any test data sent to GCM.
GMP_CLUSTER=$TAG_NAME
if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then
if [[ -n "${GCM_SECRET:-}" ]]; then
echo ">>> using GCM_SECRET credentials; running with GCM validation with explicit credentials"
PROJECT_ID=$(echo "${GCM_SECRET}" | jq -r '.project_id')
export GOOGLE_APPLICATION_CREDENTIALS=""
elif [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then
echo ">>> using GOOGLE_APPLICATION_CREDENTIALS credentials; running with GCM validation"
PROJECT_ID=$(jq -r '.project_id' "${GOOGLE_APPLICATION_CREDENTIALS}")
export GCM_EXPORT=""
else
echo ">>> no credentials specified. running without GCM validation"
TEST_ARGS="${TEST_ARGS} -skip-gcm"
Expand Down Expand Up @@ -123,6 +129,7 @@ connect_registry
echo ">>> executing gmp e2e tests: ${GO_TEST}"
# Note: a test failure here should exit non-zero and leave the cluster running
# for debugging.

go test -v -timeout 10m "${REPO_ROOT}/e2e" -run "${GO_TEST:-.}" -args ${TEST_ARGS}

# Delete cluster if it's not set to clean up post-test.
Expand Down
Loading