Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions cmd/cluster-authentication-operator-tests-ext/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"fmt"
"os"
"time"

"github.com/spf13/cobra"
"k8s.io/component-base/cli"
Expand All @@ -13,6 +14,12 @@ import (
"github.com/openshift/cluster-authentication-operator/pkg/version"

_ "github.com/openshift/cluster-authentication-operator/test/e2e"
_ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption"
// TODO: Uncomment when e2e-encryption-kms is migrated to Ginkgo format
// _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-kms"
_ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-perf"
_ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-rotation"
_ "github.com/openshift/cluster-authentication-operator/test/e2e-oidc"

"k8s.io/klog/v2"
)
Expand Down Expand Up @@ -58,6 +65,17 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) {
registry := oteextension.NewRegistry()
extension := oteextension.NewExtension("openshift", "payload", "cluster-authentication-operator")

// The following suite runs tests that verify the operator's behaviour.
// This suite is executed only on pull requests targeting this repository.
// Tests tagged with [Parallel] and any of [Operator], [OIDC], [Templates], [Tokens] are included in this suite.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator/parallel",
Parallelism: 4,
Qualifiers: []string{
`name.contains("[Parallel]") && (name.contains("[Operator]") || name.contains("[OIDC]") || name.contains("[Templates]") || name.contains("[Tokens]"))`,
},
})

// The following suite runs tests that verify the operator's behaviour.
// This suite is executed only on pull requests targeting this repository.
// Tests tagged with [Serial] and any of [Operator], [OIDC], [Templates], [Tokens] are included in this suite.
Expand All @@ -69,6 +87,70 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) {
},
})

// The following suite runs basic encryption tests that modify cluster-wide encryption configuration.
// These tests must run serially as they configure encryption settings.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator-encryption/serial",
Parallelism: 1,
Qualifiers: []string{
`name.contains("[Encryption]") && name.contains("[Serial]") && !name.contains("Rotation") && !name.contains("Perf") && !name.contains("KMS")`,
},
})

// The following suite runs encryption rotation tests.
// These tests must run serially as they configure encryption settings.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator-encryption-rotation/serial",
Parallelism: 1,
Qualifiers: []string{
`name.contains("[Encryption]") && name.contains("[Serial]") && name.contains("Rotation")`,
},
})

// The following suite runs encryption performance tests.
// These tests must run serially as they configure encryption settings and measure performance.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator-encryption-perf/serial",
Parallelism: 1,
Qualifiers: []string{
`name.contains("[Encryption]") && name.contains("[Serial]") && name.contains("Perf")`,
},
})

// The following suite runs KMS encryption tests.
// These tests must run serially as they configure KMS encryption settings.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator-encryption-kms/serial",
Parallelism: 1,
Qualifiers: []string{
`name.contains("[Encryption]") && name.contains("[Serial]") && name.contains("KMS")`,
},
})

// The following suite runs OIDC-specific tests.
// These tests can run in parallel.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/oidc/parallel",
Parallelism: 4,
Qualifiers: []string{
`name.contains("[OIDC]") && name.contains("[Parallel]")`,
},
})

// The following suite runs OIDC-specific disruptive tests.
// These tests must run serially as they modify cluster authentication configuration
// and may disrupt cluster operations.
defaultTimeout := 120 * time.Minute
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/oidc/serial-disruptive",
Parallelism: 1,
ClusterStability: oteextension.ClusterStabilityDisruptive,
TestTimeout: &defaultTimeout,
Qualifiers: []string{
`name.contains("[OIDC]") && name.contains("[Serial]") && name.contains("[Disruptive]")`,
},
})

specs, err := oteginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
if err != nil {
return nil, fmt.Errorf("couldn't build extension test specs from ginkgo: %w", err)
Expand Down
114 changes: 114 additions & 0 deletions test/e2e-encryption-perf/encryption_perf.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package e2e_encryption_perf

import (
"context"
"errors"
"fmt"
"testing"
"time"

g "github.com/onsi/ginkgo/v2"
"github.com/stretchr/testify/require"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"

configv1 "github.com/openshift/api/config/v1"
oauthapiv1 "github.com/openshift/api/oauth/v1"
operatorv1 "github.com/openshift/api/operator/v1"
oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1"
operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library"
operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption"
library "github.com/openshift/library-go/test/library/encryption"
)

const (
tokenStatsKey = "created oauthaccesstokens"
)

var _ = g.Describe("[sig-auth] authentication operator", func() {
g.It("[Encryption][Serial] TestPerfEncryptionTypeAESCBC", func() {
testPerfEncryptionTypeAESCBC(g.GinkgoTB())
})
})

func testPerfEncryptionTypeAESCBC(tt testing.TB) {
ctx := context.TODO()
clientSet := getPerfClients(tt)
operatorlibrary.TestPerfEncryption(tt, library.PerfScenario{
BasicScenario: library.BasicScenario{
Namespace: "openshift-config-managed",
LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver",
EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"),
EncryptionConfigSecretNamespace: "openshift-config-managed",
OperatorNamespace: "openshift-authentication-operator",
TargetGRs: operatorencryption.DefaultTargetGRs,
AssertFunc: operatorencryption.AssertTokens,
},
GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) {
apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
return nil, err
}
return apiServerOperator.Status.Conditions, nil
},
AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) {
tokenCount, ok := statStore[tokenStatsKey]
if !ok {
err := errors.New("missing oauth access tokens count stats, can't continue the test")
require.NoError(t, err)
}
if tokenCount < 14000 {
err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount)
require.NoError(t, err)
}
t.Logf("Created %d access tokens", tokenCount)
},
AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) {
t.Logf("migration took %v", migrationTime)
expectedMigrationTime := 10 * time.Minute
if migrationTime > expectedMigrationTime {
t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime)
}
},
DBLoaderWorkers: 3,
DBLoaderFunc: library.DBLoaderRepeat(1, false,
library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)),
EncryptionProvider: configv1.EncryptionType("aescbc"),
})
}

func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType {
return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error {
_, tokenNameHash := operatorlibrary.GenerateOAuthTokenPair()
token := &oauthapiv1.OAuthAccessToken{
ObjectMeta: metav1.ObjectMeta{
Name: tokenNameHash,
},
RefreshToken: "I have no special talents. I am only passionately curious",
UserName: "kube:admin",
Scopes: []string{"user:full"},
RedirectURI: "redirect.me.to.token.of.life",
ClientName: "console",
UserUID: "non-existing-user-id",
}
_, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{})
return err
}
}

func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error {
statsCollector(tokenStatsKey)
return nil
}

func getPerfClients(t testing.TB) operatorencryption.ClientSet {
t.Helper()

kubeConfig := operatorlibrary.NewClientConfigForTest(t)

kubeConfig.QPS = 300
kubeConfig.Burst = 600

return operatorencryption.GetClientsFor(t, kubeConfig)
}
100 changes: 1 addition & 99 deletions test/e2e-encryption-perf/encryption_perf_test.go
Original file line number Diff line number Diff line change
@@ -1,107 +1,9 @@
package e2e_encryption_perf

import (
"context"
"errors"
"fmt"
"testing"
"time"

"github.com/stretchr/testify/require"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"

configv1 "github.com/openshift/api/config/v1"
oauthapiv1 "github.com/openshift/api/oauth/v1"
operatorv1 "github.com/openshift/api/operator/v1"
oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1"
operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library"
operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption"
library "github.com/openshift/library-go/test/library/encryption"
)

const (
tokenStatsKey = "created oauthaccesstokens"
)

func TestPerfEncryptionTypeAESCBC(tt *testing.T) {
ctx := context.TODO()
clientSet := getPerfClients(tt)
library.TestPerfEncryption(tt, library.PerfScenario{
BasicScenario: library.BasicScenario{
Namespace: "openshift-config-managed",
LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver",
EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"),
EncryptionConfigSecretNamespace: "openshift-config-managed",
OperatorNamespace: "openshift-authentication-operator",
TargetGRs: operatorencryption.DefaultTargetGRs,
AssertFunc: operatorencryption.AssertTokens,
},
GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) {
apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
return nil, err
}
return apiServerOperator.Status.Conditions, nil
},
AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) {
tokenCount, ok := statStore[tokenStatsKey]
if !ok {
err := errors.New("missing oauth access tokens count stats, can't continue the test")
require.NoError(t, err)
}
if tokenCount < 14000 {
err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount)
require.NoError(t, err)
}
t.Logf("Created %d access tokens", tokenCount)
},
AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) {
t.Logf("migration took %v", migrationTime)
expectedMigrationTime := 10 * time.Minute
if migrationTime > expectedMigrationTime {
t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime)
}
},
DBLoaderWorkers: 3,
DBLoaderFunc: library.DBLoaderRepeat(1, false,
library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)),
EncryptionProvider: configv1.EncryptionType("aescbc"),
})
}

func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType {
return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error {
_, tokenNameHash := operatorlibrary.GenerateOAuthTokenPair()
token := &oauthapiv1.OAuthAccessToken{
ObjectMeta: metav1.ObjectMeta{
Name: tokenNameHash,
},
RefreshToken: "I have no special talents. I am only passionately curious",
UserName: "kube:admin",
Scopes: []string{"user:full"},
RedirectURI: "redirect.me.to.token.of.life",
ClientName: "console",
UserUID: "non-existing-user-id",
}
_, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{})
return err
}
}

func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error {
statsCollector(tokenStatsKey)
return nil
}

func getPerfClients(t *testing.T) operatorencryption.ClientSet {
t.Helper()

kubeConfig := operatorlibrary.NewClientConfigForTest(t)

kubeConfig.QPS = 300
kubeConfig.Burst = 600

return operatorencryption.GetClientsFor(t, kubeConfig)
testPerfEncryptionTypeAESCBC(tt)
}
Loading