verify histogram with measured provision time

go.mod

@@ -1,8 +1,8 @@
 module github.com/codeready-toolchain/toolchain-e2e
 
 require (
-	github.com/codeready-toolchain/api v0.0.0-20250131222557-beba5463f429
-	github.com/codeready-toolchain/toolchain-common v0.0.0-20250225125153-139ef30bbd6d
+	github.com/codeready-toolchain/api v0.0.0-20250227073728-5999971adb48
+	github.com/codeready-toolchain/toolchain-common v0.0.0-20250227105612-03e9ebab49c7
 	github.com/davecgh/go-spew v1.1.1
 	github.com/fatih/color v1.15.0
 	github.com/ghodss/yaml v1.0.0

go.sum

@@ -119,10 +119,10 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/codeready-toolchain/api v0.0.0-20250131222557-beba5463f429 h1:I7xzHRmstvzucH9NqF54xvsM/yYuWOp/G5+BUr5j4tY=
-github.com/codeready-toolchain/api v0.0.0-20250131222557-beba5463f429/go.mod h1:gPwicZPTmRm1PF75ysEYXaYKdXoFgwgCggTJd1oYmOs=
-github.com/codeready-toolchain/toolchain-common v0.0.0-20250225125153-139ef30bbd6d h1:iEUmJ9XB7Q6BUWG76ocuECZBPIY2wDxQhAXTAW3CM8g=
-github.com/codeready-toolchain/toolchain-common v0.0.0-20250225125153-139ef30bbd6d/go.mod h1:/PeschEEyTZIAsXQxIf83Fl8kOpSOHDfR5yyanz+bSA=
+github.com/codeready-toolchain/api v0.0.0-20250227073728-5999971adb48 h1:jqGcYw4KdQzqe5WEp+06HXBRyosAktgO5Y6ADs+NF5A=
+github.com/codeready-toolchain/api v0.0.0-20250227073728-5999971adb48/go.mod h1:gPwicZPTmRm1PF75ysEYXaYKdXoFgwgCggTJd1oYmOs=
+github.com/codeready-toolchain/toolchain-common v0.0.0-20250227105612-03e9ebab49c7 h1:Y4EMaYLn62ptrChtaPcqzGFe0h0TNDVmwL1WOVbu2IY=
+github.com/codeready-toolchain/toolchain-common v0.0.0-20250227105612-03e9ebab49c7/go.mod h1:mNRKnxjBHxggT03t6TwGJT5AZvmHMn6oQ32DrkGwV+w=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

test/metrics/metrics_test.go

@@ -12,9 +12,11 @@ import (
 	"github.com/codeready-toolchain/toolchain-common/pkg/hash"
 	"github.com/codeready-toolchain/toolchain-common/pkg/states"
 	testconfig "github.com/codeready-toolchain/toolchain-common/pkg/test/config"
+	testSpc "github.com/codeready-toolchain/toolchain-common/pkg/test/spaceprovisionerconfig"
 	. "github.com/codeready-toolchain/toolchain-e2e/testsupport"
 	authsupport "github.com/codeready-toolchain/toolchain-e2e/testsupport/auth"
 	"github.com/codeready-toolchain/toolchain-e2e/testsupport/cleanup"
+	"github.com/codeready-toolchain/toolchain-e2e/testsupport/spaceprovisionerconfig"
 	"github.com/codeready-toolchain/toolchain-e2e/testsupport/wait"
 
 	"github.com/google/uuid"
@@ -135,6 +137,7 @@ func TestMetricsWhenUsersManuallyApprovedAndThenDeactivated(t *testing.T) {
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsDeactivatedMetric, 0)                                                 // none deactivated
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 1, "cluster_name", memberAwait.ClusterName)                        // 1 space created on member-1
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 2, "cluster_name", memberAwait2.ClusterName)                       // 2 spaces created on member-2
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 0)                                        // no tracking of the provision time for manual approval
 
 	// check if the MUR and Space counts match in ToolchainStatus
 	_, err = hostAwait.WaitForToolchainStatus(t,
@@ -171,6 +174,7 @@ func TestMetricsWhenUsersManuallyApprovedAndThenDeactivated(t *testing.T) {
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsDeactivatedMetric, 2)                                                 // two deactivated
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 1, "cluster_name", memberAwait.ClusterName)                        // 1 space still in member-1
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait2.ClusterName)                       // 2 spaces deleted from member-2
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 0)                                        // no change when deactivated
 
 	// check if the MUR and Space counts match in ToolchainStatus
 	_, err = hostAwait.WaitForToolchainStatus(t,
@@ -219,7 +223,7 @@ func TestMetricsWhenUsersAutomaticallyApprovedAndThenDeactivated(t *testing.T) {
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 2, "method", "automatic")                   // both automatically approved
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "manual")                      // not manually approved
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsDeactivatedMetric, 0)                                                 // none deactivated
-
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 2)                                        // one record for both automatically approved users
 	// when deactivating the users
 	for username, usersignup := range usersignups {
 		_, err := wait.For(t, hostAwait.Awaitility, &toolchainv1alpha1.UserSignup{}).
@@ -244,6 +248,7 @@ func TestMetricsWhenUsersAutomaticallyApprovedAndThenDeactivated(t *testing.T) {
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 2, "method", "automatic")                   // all deactivated (but counters are never decremented)
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "manual")                      // all deactivated (but counters are never decremented)
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsDeactivatedMetric, 2)                                                 // all deactivated
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 2)                                        // no change when deactivated
 }
 
 // TestVerificationRequiredMetric verifies that `UserSignupVerificationRequiredMetric` counters are increased only once when users are created/reactivated
@@ -307,7 +312,8 @@ func TestVerificationRequiredMetric(t *testing.T) {
 			require.NotEmpty(t, verificationCode)
 			// Attempt to verify with an incorrect verification code
 			NewHTTPRequest(t).InvokeEndpoint("GET", route+"/api/v1/signup/verification/invalid", token0, "", http.StatusForbidden)
-			hostAwait.WaitForMetricDelta(t, wait.UserSignupVerificationRequiredMetric, 1) // no change after verification initiated
+			hostAwait.WaitForMetricDelta(t, wait.UserSignupVerificationRequiredMetric, 1)  // no change after verification initiated
+			hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 0) // no tracking of provision time for users with phone verification step
 		})
 
 		t.Run("no change to metric when user deactivated", func(t *testing.T) {
@@ -530,6 +536,7 @@ func TestMetricsWhenUsersBanned(t *testing.T) {
 	hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 0, "domain", "internal")
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait.ClusterName)
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait2.ClusterName)
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 0) // manual approval and banning is not recorded in provisioned time
 
 	t.Run("unban the banned user", func(t *testing.T) {
 		// when unbaning the user
@@ -555,6 +562,7 @@ func TestMetricsWhenUsersBanned(t *testing.T) {
 		hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 0, "domain", "internal")
 		hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 1, "cluster_name", memberAwait.ClusterName)  // space provisioned on member1
 		hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait2.ClusterName) // no spaces on member2
+		hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 0)                  // unbanning manually approved user is not recorded in provisioned time
 	})
 }
 
@@ -565,34 +573,37 @@ func TestMetricsWhenUserDisabled(t *testing.T) {
 	hostAwait := awaitilities.Host()
 	memberAwait := awaitilities.Member1()
 	memberAwait2 := awaitilities.Member2()
-	hostAwait.UpdateToolchainConfig(t, testconfig.AutomaticApproval().Enabled(false))
+	hostAwait.UpdateToolchainConfig(t,
+		testconfig.AutomaticApproval().Enabled(true),
+		testconfig.RegistrationService().Verification().Enabled(false))
 	// host metrics should be available at this point
 	hostAwait.InitMetrics(t, awaitilities.Member1().ClusterName, awaitilities.Member2().ClusterName)
 	t.Cleanup(func() {
 		t.Log("waiting for metrics to get back to their baseline values...")
 		hostAwait.WaitForMetricBaseline(t, wait.SpacesMetric, "cluster_name", memberAwait.ClusterName)  // wait until counter is back to 0
 		hostAwait.WaitForMetricBaseline(t, wait.SpacesMetric, "cluster_name", memberAwait2.ClusterName) // wait until counter is back to 0
 	})
+	// disable member2 so the user is provisioned to member1
+	spaceprovisionerconfig.UpdateForCluster(t, hostAwait.Awaitility, memberAwait2.ClusterName, testSpc.Enabled(false))
 
 	// Create UserSignup
 	user := NewSignupRequest(awaitilities).
 		Username("janedoe").
-		ManuallyApprove().
-		TargetCluster(memberAwait).
 		EnsureMUR().
-		RequireConditions(wait.ConditionSet(wait.Default(), wait.ApprovedByAdmin())...).
+		RequireConditions(wait.ConditionSet(wait.Default(), wait.ApprovedAutomatically())...).
 		Execute(t)
 	mur := user.MUR
 
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsMetric, 1)
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedMetric, 1)                                  // approved
-	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "automatic") // not automatically approved
-	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 1, "method", "manual")    // manually approved
+	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 1, "method", "automatic") // automatically approved
+	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "manual")    // not manually approved
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsBannedMetric, 0)
 	hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 0, "domain", "internal")
 	hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 1, "domain", "external")
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 1, "cluster_name", memberAwait.ClusterName)  // space present on member1
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait2.ClusterName) // no space on member2
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 1)                  // automatically approved user is measured
 
 	// when disabling MUR
 	_, err := wait.For(t, hostAwait.Awaitility, &toolchainv1alpha1.MasterUserRecord{}).
@@ -606,13 +617,14 @@ func TestMetricsWhenUserDisabled(t *testing.T) {
 	// verify the metrics
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsMetric, 1)
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedMetric, 1)                                  // still approved even though (temporarily) disabled
-	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "automatic") // not automatically approved
-	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 1, "method", "manual")    // manually approved
+	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 1, "method", "automatic") // automatically approved
+	hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "manual")    // not manually approved
 	hostAwait.WaitForMetricDelta(t, wait.UserSignupsBannedMetric, 0)
 	hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 0, "domain", "internal")
 	hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 1, "domain", "external")
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 1, "cluster_name", memberAwait.ClusterName)  // space is on member1
 	hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait2.ClusterName) // no space on member2
+	hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 1)                  // disabling has no impact
 
 	t.Run("re-enabled mur", func(t *testing.T) {
 		// When re-enabling MUR
@@ -627,13 +639,14 @@ func TestMetricsWhenUserDisabled(t *testing.T) {
 		// verify the metrics
 		hostAwait.WaitForMetricDelta(t, wait.UserSignupsMetric, 1)                                          // unchanged, user was already provisioned
 		hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedMetric, 1)                                  // unchanged, user was already provisioned
-		hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "automatic") // unchanged, user was already provisioned
-		hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 1, "method", "manual")    // unchanged, user was already provisioned
+		hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 1, "method", "automatic") // unchanged, user was already provisioned
+		hostAwait.WaitForMetricDelta(t, wait.UserSignupsApprovedWithMethodMetric, 0, "method", "manual")    // unchanged, user was already provisioned
 		hostAwait.WaitForMetricDelta(t, wait.UserSignupsBannedMetric, 0)
 		hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 0, "domain", "internal")
 		hostAwait.WaitForMetricDelta(t, wait.MasterUserRecordsPerDomainMetric, 1, "domain", "external") // unchanged, user was already provisioned
 		hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 1, "cluster_name", memberAwait.ClusterName)  // unchanged, user was already provisioned
 		hostAwait.WaitForMetricDelta(t, wait.SpacesMetric, 0, "cluster_name", memberAwait2.ClusterName)
+		hostAwait.WaitForHistogramInfBucketDelta(t, wait.SignupProvisionTimeMetric, 1) // re-enabling has no impact
 	})
 }
 

testsupport/metrics/metrics.go

@@ -14,8 +14,24 @@ import (
 )
 
 func GetMetricValue(restConfig *rest.Config, url string, family string, expectedLabels []string) (float64, error) {
+	value, err := getMetricValue(restConfig, url, family, expectedLabels, getValue)
+	if value == nil {
+		return -1, err
+	}
+	return *value, err
+}
+
+func GetHistogramBuckets(restConfig *rest.Config, url string, family string, expectedLabels []string) ([]*dto.Bucket, error) {
+	value, err := getMetricValue(restConfig, url, family, expectedLabels, getBuckets)
+	if value == nil {
+		return nil, err
+	}
+	return *value, err
+}
+
+func getMetricValue[T any](restConfig *rest.Config, url string, family string, expectedLabels []string, getValue func(dto.MetricType, *dto.Metric) (*T, error)) (*T, error) {
 	if len(expectedLabels)%2 != 0 {
-		return -1, fmt.Errorf("received odd number of label arguments, labels must be key-value pairs")
+		return nil, fmt.Errorf("received odd number of label arguments, labels must be key-value pairs")
 	}
 	uri := fmt.Sprintf("https://%s/metrics", url)
 	var metrics []byte
@@ -28,28 +44,28 @@ func GetMetricValue(restConfig *rest.Config, url string, family string, expected
 	}
 	request, err := http.NewRequest("Get", uri, nil)
 	if err != nil {
-		return -1, err
+		return nil, err
 	}
 	if restConfig.BearerToken != "" {
 		request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", restConfig.BearerToken))
 	}
 	resp, err := client.Do(request)
 	if err != nil {
-		return -1, err
+		return nil, err
 	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
 	metrics, err = io.ReadAll(resp.Body)
 	if err != nil {
-		return -1, err
+		return nil, err
 	}
 
 	// parse the metrics
 	parser := expfmt.TextParser{}
 	families, err := parser.TextToMetricFamilies(bytes.NewReader(metrics))
 	if err != nil {
-		return -1, err
+		return nil, err
 	}
 	for _, f := range families {
 		if f.GetName() == family {
@@ -82,19 +98,32 @@ func GetMetricValue(restConfig *rest.Config, url string, family string, expected
 		}
 	}
 	// here we can return `0` is the metric does not exist, which may be valid if the expected value is `0`, too.
-	return 0, fmt.Errorf("metric '%s{%v}' not found", family, expectedLabels)
+	return new(T), fmt.Errorf("metric '%s{%v}' not found", family, expectedLabels)
 }
 
-func getValue(t dto.MetricType, m *dto.Metric) (float64, error) {
+func getValue(t dto.MetricType, m *dto.Metric) (*float64, error) {
 	switch t { // nolint:exhaustive
 	case dto.MetricType_COUNTER:
-		return m.GetCounter().GetValue(), nil
+		value := m.GetCounter().GetValue()
+		return &value, nil
 	case dto.MetricType_GAUGE:
-		return m.GetGauge().GetValue(), nil
+		value := m.GetGauge().GetValue()
+		return &value, nil
 	case dto.MetricType_UNTYPED:
-		return m.GetUntyped().GetValue(), nil
+		value := m.GetUntyped().GetValue()
+		return &value, nil
+	default:
+		return nil, fmt.Errorf("unknown or unsupported metric type %s", t.String())
+	}
+}
+
+func getBuckets(t dto.MetricType, m *dto.Metric) (*[]*dto.Bucket, error) {
+	switch t { // nolint:exhaustive
+	case dto.MetricType_HISTOGRAM:
+		value := m.GetHistogram().GetBucket()
+		return &value, nil
 	default:
-		return -1, fmt.Errorf("unknown or unsupported metric type %s", t.String())
+		return nil, fmt.Errorf("unknown or unsupported metric type %s", t.String())
 	}
 }