diff --git a/pkg/operator/encryption/controllers/key_controller.go b/pkg/operator/encryption/controllers/key_controller.go index c999f140f0..ef0373db43 100644 --- a/pkg/operator/encryption/controllers/key_controller.go +++ b/pkg/operator/encryption/controllers/key_controller.go @@ -27,6 +27,7 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/encryption/crypto" + "github.com/openshift/library-go/pkg/operator/encryption/kms" "github.com/openshift/library-go/pkg/operator/encryption/secrets" "github.com/openshift/library-go/pkg/operator/encryption/state" "github.com/openshift/library-go/pkg/operator/encryption/statemachine" @@ -266,6 +267,14 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, InternalReason: internalReason, ExternalReason: externalReason, } + if currentMode == state.KMS { + ks.KMSConfiguration = &apiserverv1.KMSConfiguration{ + APIVersion: "v2", + Name: fmt.Sprintf("%d", keyID), // this will be updated by inserting resource name + Endpoint: kms.DefaultEndpoint, + Timeout: &metav1.Duration{Duration: kms.DefaultTimeout}, + } + } return secrets.FromKeyState(c.instanceName, ks) } @@ -287,7 +296,7 @@ func (c *keyController) getCurrentModeAndExternalReason(ctx context.Context) (st reason := encryptionConfig.Encryption.Reason switch currentMode := state.Mode(apiServer.Spec.Encryption.Type); currentMode { - case state.AESCBC, state.AESGCM, state.Identity: // secretbox is disabled for now + case state.AESCBC, state.AESGCM, state.KMS, state.Identity: // secretbox is disabled for now return currentMode, reason, nil case "": // unspecified means use the default (which can change over time) return state.DefaultMode, reason, nil @@ -341,6 +350,19 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern return 0, "", false } + if currentMode == state.KMS { + // We are here because Encryption Mode is not changed + + // For now in v1, we don't support configurational changes. Therefore, + // it is pointless comparing the secrets. + + // For KMS mode, we don't do time-based rotation. Therefore, we shortcut here + // KMS keys are rotated externally by the KMS system. + // Moreover, we don't trigger new key when external reason is changed. + // Because it would lead to duplicate providers which is not allowed. + return 0, "", false + } + // if the most recent secret has a different external reason than the current reason, we need to generate a new key if latestKey.ExternalReason != externalReason && len(externalReason) != 0 { return latestKeyID, "external-reason-changed", true diff --git a/pkg/operator/encryption/controllers/key_controller_test.go b/pkg/operator/encryption/controllers/key_controller_test.go index 03ba8c45e4..f681ba18c5 100644 --- a/pkg/operator/encryption/controllers/key_controller_test.go +++ b/pkg/operator/encryption/controllers/key_controller_test.go @@ -41,6 +41,9 @@ func TestKeyController(t *testing.T) { apiServerWithAESGCM := simpleAPIServer.DeepCopy() apiServerWithAESGCM.Spec.Encryption = configv1.APIServerEncryption{Type: "aesgcm"} + apiServerWithKMS := simpleAPIServer.DeepCopy() + apiServerWithKMS.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS"} + scenarios := []struct { name string initialObjects []runtime.Object @@ -324,6 +327,246 @@ func TestKeyController(t *testing.T) { } }, }, + + { + name: "checks if a KMS secret is created when KMS encryption is enabled", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + }, + apiServerObjects: []runtime.Object{apiServerWithKMS}, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + + // Verify mode annotation is KMS + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"] != "KMS" { + ts.Errorf("expected mode to be KMS, got %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"]) + } + + // Verify KMS config annotation exists + kmsConfig := actualSecret.Annotations["encryption.apiserver.operator.openshift.io/kms-config"] + if kmsConfig == "" { + ts.Error("expected kms-config annotation to be present") + } + if kmsConfig != `{"apiVersion":"v2","name":"1","endpoint":"unix:///var/run/kmsplugin/kms.sock","timeout":"10s"}` { + ts.Errorf("unexpected kms-config: %s", kmsConfig) + } + + // Verify internal reason + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-key-does-not-exist" { + ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) + } + + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, + + { + name: "no-op when a valid KMS write key exists", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 1), + }, + apiServerObjects: []runtime.Object{apiServerWithKMS}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed"}, + }, + + { + name: "creates a new KMS key when switching from AESCBC to KMS", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("61def964fb967f5d7c44a2af8dab6865"), "aescbc"), + }, + apiServerObjects: []runtime.Object{apiServerWithKMS}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + + // Verify mode changed to KMS + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"] != "KMS" { + ts.Errorf("expected mode to be KMS, got %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"]) + } + + // Verify KMS config annotation exists + kmsConfig := actualSecret.Annotations["encryption.apiserver.operator.openshift.io/kms-config"] + if kmsConfig != `{"apiVersion":"v2","name":"6","endpoint":"unix:///var/run/kmsplugin/kms.sock","timeout":"10s"}` { + ts.Errorf("unexpected kms-config: %s", kmsConfig) + } + + // Verify internal reason is mode changed + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-encryption-mode-changed" { + ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) + } + + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, + + { + name: "no-op when KMS key is migrated but not expired (no time-based rotation for KMS)", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateExpiredMigratedEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 6), + }, + apiServerObjects: []runtime.Object{apiServerWithKMS}, + targetNamespace: "kms", + // Should be no-op because KMS keys don't have time-based rotation + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed"}, + }, + { + name: "no-op when latest KMS key is not migrated yet", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 3), + }, + apiServerObjects: []runtime.Object{apiServerWithKMS}, + targetNamespace: "kms", + // Should be no-op because migration hasn't completed yet + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed"}, + }, + + { + name: "creates a new KMS key when switching from Identity to KMS", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("identity-key"), "identity"), + }, + apiServerObjects: []runtime.Object{apiServerWithKMS}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + + // Verify mode changed to KMS + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"] != "KMS" { + ts.Errorf("expected mode to be KMS, got %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"]) + } + + // Verify KMS config annotation exists + kmsConfig := actualSecret.Annotations["encryption.apiserver.operator.openshift.io/kms-config"] + if kmsConfig != `{"apiVersion":"v2","name":"6","endpoint":"unix:///var/run/kmsplugin/kms.sock","timeout":"10s"}` { + ts.Errorf("unexpected kms-config: %s", kmsConfig) + } + + // Verify internal reason is mode changed + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-encryption-mode-changed" { + ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) + } + + // Verify key ID incremented + if actualSecret.Name != "encryption-key-kms-6" { + ts.Errorf("expected key ID 6, got %s", actualSecret.Name) + } + + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, + + { + name: "creates a new AESCBC key when switching from KMS to AESCBC", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialObjects: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 7), + }, + apiServerObjects: []runtime.Object{apiServerWithAESCBC}, + targetNamespace: "kms", + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + + // Verify mode changed to aescbc + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"] != "aescbc" { + ts.Errorf("expected mode to be aescbc, got %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/mode"]) + } + + // Verify KMS config annotation is removed (not present for AESCBC) + if kmsConfig, exists := actualSecret.Annotations["encryption.apiserver.operator.openshift.io/kms-config"]; exists { + ts.Errorf("expected kms-config annotation to be absent, got: %s", kmsConfig) + } + + // Verify internal reason is mode changed + if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-encryption-mode-changed" { + ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) + } + + // Verify key ID incremented to 8 + if actualSecret.Name != "encryption-key-kms-8" { + ts.Errorf("expected key ID 8, got %s", actualSecret.Name) + } + + // Verify it's a valid 32-byte AES key + if err := encryptiontesting.ValidateEncryptionKey(actualSecret); err != nil { + ts.Error(err) + } + + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, } for _, scenario := range scenarios { @@ -444,6 +687,7 @@ func TestGetCurrentModeAndExternalReason(t *testing.T) { prefix []string apiServerObjects []runtime.Object expectedReasonFromCfg string + expectedKMSConfig []byte }{ { name: "no prefix provided, flat observed config", @@ -478,6 +722,12 @@ func TestGetCurrentModeAndExternalReason(t *testing.T) { name: "reading empty config works", apiServerObjects: []runtime.Object{&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}}}, }, + + { + name: "kms encryption mode", + apiServerObjects: []runtime.Object{&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, Spec: configv1.APIServerSpec{Encryption: configv1.APIServerEncryption{Type: "KMS"}}}}, + expectedKMSConfig: []byte("unix:///var/run/kmsplugin/kms.sock"), + }, } for _, scenario := range scenarios { diff --git a/pkg/operator/encryption/controllers/state_controller_test.go b/pkg/operator/encryption/controllers/state_controller_test.go index 46fc507c5c..9a3bab0ad7 100644 --- a/pkg/operator/encryption/controllers/state_controller_test.go +++ b/pkg/operator/encryption/controllers/state_controller_test.go @@ -5,10 +5,13 @@ import ( "encoding/base64" "errors" "fmt" - clocktesting "k8s.io/utils/clock/testing" "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + clocktesting "k8s.io/utils/clock/testing" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -707,6 +710,405 @@ func TestStateController(t *testing.T) { encryptiontesting.ValidateOperatorClientConditions(ts, operatorClient, []operatorv1.OperatorCondition{expectedCondition}) }, }, + + // scenario 12: KMS secret exists => encryption config created without write key + { + name: "KMS: secret with EncryptionConfig is created without a write key", + targetNamespace: "kms", + encryptionSecretSelector: metav1.ListOptions{LabelSelector: "encryption.apiserver.operator.openshift.io/component=kms"}, + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialResources: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1), + }, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms", "create:events:kms"}, + expectedEncryptionCfg: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }}, + }}, + }, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, destName string, expectedEncryptionCfg *apiserverconfigv1.EncryptionConfiguration) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + err := validateSecretWithEncryptionConfig(actualSecret, expectedEncryptionCfg, destName) + if err != nil { + ts.Fatalf("failed to verfy the encryption config, due to %v", err) + } + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, + + // scenario 13: migrated KMS key => encryption config with KMS write key + { + name: "KMS: secret with EncryptionConfig is created with KMS as write key", + targetNamespace: "kms", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialResources: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1, time.Now()), + func() *corev1.Secret { + ec := &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }}, + }}, + } + ecs := createEncryptionCfgSecret(t, "kms", "1", ec) + return ecs + }(), + }, + expectedEncryptionCfg: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + expectedActions: []string{ + "list:pods:kms", + "get:secrets:kms", + "list:secrets:openshift-config-managed", + "get:secrets:openshift-config-managed", + "create:secrets:openshift-config-managed", + "create:events:kms", + "create:events:kms", + }, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, destName string, expectedEncryptionCfg *apiserverconfigv1.EncryptionConfiguration) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("create", "secrets") { + createAction := action.(clientgotesting.CreateAction) + actualSecret := createAction.GetObject().(*corev1.Secret) + err := validateSecretWithEncryptionConfig(actualSecret, expectedEncryptionCfg, destName) + if err != nil { + ts.Fatalf("failed to verfy the encryption config, due to %v", err) + } + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, + + // scenario 14: no-op when KMS config is stable + { + name: "KMS: no-op when no key is transitioning", + targetNamespace: "kms", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialResources: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateMigratedEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1, time.Now()), + func() *corev1.Secret { + ec := &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + } + ecs := createEncryptionCfgSecret(t, "kms", "1", ec) + return ecs + }(), + func() *corev1.Secret { + ec := &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + } + ecs := createEncryptionCfgSecret(t, "openshift-config-managed", "1", ec) + ecs.Name = "encryption-config-kms" + return ecs + }(), + }, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config-managed"}, + }, + + // scenario 15: KMS key transitioning => new KMS write key set + { + name: "KMS: new KMS key is transitioning (observed as a read key) so it is used as a write key", + targetNamespace: "kms", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialResources: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateExpiredMigratedEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 2), + func() *corev1.Secret { // encryption config in kms namespace + ec := &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + } + ecs := createEncryptionCfgSecret(t, "kms", "1", ec) + return ecs + }(), + func() *corev1.Secret { // encryption config in openshift-config-managed + ec := &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + } + ecs := createEncryptionCfgSecret(t, "openshift-config-managed", "1", ec) + ecs.Name = "encryption-config-kms" + return ecs + }(), + }, + expectedEncryptionCfg: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + expectedActions: []string{ + "list:pods:kms", + "get:secrets:kms", + "list:secrets:openshift-config-managed", + "get:secrets:openshift-config-managed", + "update:secrets:openshift-config-managed", + "create:events:kms", + "create:events:kms", + }, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, destName string, expectedEncryptionCfg *apiserverconfigv1.EncryptionConfiguration) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("update", "secrets") { + updateAction := action.(clientgotesting.UpdateAction) + actualSecret := updateAction.GetObject().(*corev1.Secret) + err := validateSecretWithEncryptionConfig(actualSecret, expectedEncryptionCfg, destName) + if err != nil { + ts.Fatalf("failed to verfy the encryption config, due to %v", err) + } + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, + + // scenario 16: AESCBC→KMS migration scenario + { + name: "KMS: AESCBC to KMS migration - KMS added as read key", + targetNamespace: "kms", + targetGRs: []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + }, + initialResources: []runtime.Object{ + encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateEncryptionKeySecretWithRawKey("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1, []byte("61def964fb967f5d7c44a2af8dab6865")), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 2), + func() *corev1.Secret { // encryption config in kms namespace + keysRes := encryptiontesting.EncryptionKeysResourceTuple{ + Resource: "secrets", + Keys: []apiserverconfigv1.Key{ + { + Name: "1", + Secret: "NjFkZWY5NjRmYjk2N2Y1ZDdjNDRhMmFmOGRhYjY4NjU=", // # notsecret + }, + }, + } + ec := encryptiontesting.CreateEncryptionCfgWithWriteKey([]encryptiontesting.EncryptionKeysResourceTuple{keysRes}) + ecs := createEncryptionCfgSecret(t, "kms", "1", ec) + return ecs + }(), + func() *corev1.Secret { // encryption config in openshift-config-managed namespace + keysRes := encryptiontesting.EncryptionKeysResourceTuple{ + Resource: "secrets", + Keys: []apiserverconfigv1.Key{ + { + Name: "1", + Secret: "NjFkZWY5NjRmYjk2N2Y1ZDdjNDRhMmFmOGRhYjY4NjU=", // # notsecret + }, + }, + } + ec := encryptiontesting.CreateEncryptionCfgWithWriteKey([]encryptiontesting.EncryptionKeysResourceTuple{keysRes}) + ecs := createEncryptionCfgSecret(t, "openshift-config-managed", "1", ec) + ecs.Name = "encryption-config-kms" + return ecs + }(), + }, + expectedEncryptionCfg: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "1", + Secret: "NjFkZWY5NjRmYjk2N2Y1ZDdjNDRhMmFmOGRhYjY4NjU=", // # notsecret + }}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + expectedActions: []string{ + "list:pods:kms", + "get:secrets:kms", + "list:secrets:openshift-config-managed", + "get:secrets:openshift-config-managed", + "update:secrets:openshift-config-managed", + "create:events:kms", + "create:events:kms", + }, + validateFunc: func(ts *testing.T, actions []clientgotesting.Action, destName string, expectedEncryptionCfg *apiserverconfigv1.EncryptionConfiguration) { + wasSecretValidated := false + for _, action := range actions { + if action.Matches("update", "secrets") { + updateAction := action.(clientgotesting.UpdateAction) + actualSecret := updateAction.GetObject().(*corev1.Secret) + err := validateSecretWithEncryptionConfig(actualSecret, expectedEncryptionCfg, destName) + if err != nil { + ts.Fatalf("failed to verfy the encryption config, due to %v", err) + } + wasSecretValidated = true + break + } + } + if !wasSecretValidated { + ts.Errorf("the secret wasn't created and validated") + } + }, + }, } for _, scenario := range scenarios { @@ -800,8 +1202,10 @@ func validateSecretWithEncryptionConfig(actualSecret *corev1.Secret, expectedEnc return fmt.Errorf("failed to verfy the encryption config, due to %v", err) } - if !equality.Semantic.DeepEqual(expectedEncryptionCfg, actualEncryptionCfg) { - return fmt.Errorf("%s", diff.Diff(expectedEncryptionCfg, actualEncryptionCfg)) + // Ignore KMS Name field since it contains a random suffix + ignoreKMSName := cmpopts.IgnoreFields(apiserverconfigv1.KMSConfiguration{}, "Name") + if !cmp.Equal(expectedEncryptionCfg, actualEncryptionCfg, ignoreKMSName) { + return fmt.Errorf("%s", cmp.Diff(expectedEncryptionCfg, actualEncryptionCfg, ignoreKMSName)) } // rewrite the payload and compare the rest diff --git a/pkg/operator/encryption/crypto/keys.go b/pkg/operator/encryption/crypto/keys.go index a623d30f79..806f39f533 100644 --- a/pkg/operator/encryption/crypto/keys.go +++ b/pkg/operator/encryption/crypto/keys.go @@ -11,7 +11,8 @@ var ( state.AESCBC: NewAES256Key, state.AESGCM: NewAES256Key, state.SecretBox: NewAES256Key, // secretbox requires a 32 byte key so we can reuse the same function here - state.Identity: NewIdentityKey, + state.Identity: NewEmptyKey, + state.KMS: NewEmptyKey, } ) @@ -23,6 +24,6 @@ func NewAES256Key() []byte { return b } -func NewIdentityKey() []byte { +func NewEmptyKey() []byte { return make([]byte, 16) // the key is not used to perform encryption but must be a valid AES key } diff --git a/pkg/operator/encryption/encryptionconfig/config.go b/pkg/operator/encryption/encryptionconfig/config.go index 3082aa653f..9b079fdb90 100644 --- a/pkg/operator/encryption/encryptionconfig/config.go +++ b/pkg/operator/encryption/encryptionconfig/config.go @@ -2,7 +2,9 @@ package encryptionconfig import ( "encoding/base64" + "fmt" "sort" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -15,7 +17,7 @@ import ( ) var ( - emptyStaticIdentityKey = base64.StdEncoding.EncodeToString(crypto.NewIdentityKey()) + emptyStaticKey = base64.StdEncoding.EncodeToString(crypto.NewEmptyKey()) ) // FromEncryptionState converts state to config. @@ -25,7 +27,7 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes for gr, grKeys := range encryptionState { resourceConfigs = append(resourceConfigs, apiserverconfigv1.ResourceConfiguration{ Resources: []string{gr.String()}, // we are forced to lose data here because this API is broken - Providers: stateToProviders(grKeys), + Providers: stateToProviders(gr.Resource, grKeys), }) } @@ -97,7 +99,7 @@ func ToEncryptionState(encryptionConfig *apiserverconfigv1.EncryptionConfigurati case provider.AESGCM != nil && len(provider.AESGCM.Keys) == 1: s := state.AESGCM - if provider.AESGCM.Keys[0].Secret == emptyStaticIdentityKey { + if provider.AESGCM.Keys[0].Secret == emptyStaticKey { s = state.Identity } @@ -106,6 +108,22 @@ func ToEncryptionState(encryptionConfig *apiserverconfigv1.EncryptionConfigurati Mode: s, } + case provider.KMS != nil: + // Name and Secret must match to find our backed Secret + keyId, err := getKeyIDFromProviderName(provider.KMS.Name) + if err != nil { + klog.Warningf("skipping invalid provider name %s: %v", provider.KMS.Name, err) + continue // should never happen + } + ks = state.KeyState{ + Key: apiserverconfigv1.Key{ + Name: keyId, + // Since in v1 we don't support kms -> kms migrations, we can use empty key. + // Because we don't need to compare secrets to detect change. + Secret: emptyStaticKey, + }, + Mode: state.KMS, + } default: klog.Infof("skipping invalid provider index %d for resource %s", i, resourceConfig.Resources[0]) continue // should never happen @@ -139,7 +157,7 @@ func ToEncryptionState(encryptionConfig *apiserverconfigv1.EncryptionConfigurati // it primarily handles the conversion of KeyState to the appropriate provider config. // the identity mode is transformed into a custom aesgcm provider that simply exists to // curry the associated null key secret through the encryption state machine. -func stateToProviders(desired state.GroupResourceState) []apiserverconfigv1.ProviderConfiguration { +func stateToProviders(resource string, desired state.GroupResourceState) []apiserverconfigv1.ProviderConfiguration { allKeys := desired.ReadKeys providers := make([]apiserverconfigv1.ProviderConfiguration, 0, len(allKeys)+1) // one extra for identity @@ -192,6 +210,14 @@ func stateToProviders(desired state.GroupResourceState) []apiserverconfigv1.Prov Keys: []apiserverconfigv1.Key{key.Key}, }, }) + case state.KMS: + // In order to preserve the uniqueness, we should insert resource name + kmsCopy := key.KMSConfiguration.DeepCopy() + kmsCopy.Name = createKMSProviderName(key.Key.Name, resource) + provider := apiserverconfigv1.ProviderConfiguration{ + KMS: kmsCopy, + } + providers = append(providers, provider) default: // this should never happen because our input should always be valid klog.Infof("skipping key %s as it has invalid mode %s", key.Key.Name, key.Mode) @@ -210,3 +236,20 @@ func stateToProviders(desired state.GroupResourceState) []apiserverconfigv1.Prov return providers } + +func createKMSProviderName(keyID, resource string) string { + // Ideally we should have used keyId simply in kms provider name. + // However, this is an upstream constraint that every provider name must be unique. + // To maintain uniqueness while still allowing access to the keyId, we generate provider name in this format. + return fmt.Sprintf("%s-%s", keyID, resource) +} + +func getKeyIDFromProviderName(providerName string) (string, error) { + // We just need to obtain the keyID to find our backed secret + // e.g. "1-secrets" + parsed := strings.Split(providerName, "-") + if len(parsed) != 2 { + return "", fmt.Errorf("invalid provider name: %s", providerName) + } + return parsed[0], nil +} diff --git a/pkg/operator/encryption/encryptionconfig/config_test.go b/pkg/operator/encryption/encryptionconfig/config_test.go index 865a3d8b1d..d93989ad9e 100644 --- a/pkg/operator/encryption/encryptionconfig/config_test.go +++ b/pkg/operator/encryption/encryptionconfig/config_test.go @@ -4,8 +4,11 @@ import ( "encoding/base64" "fmt" "testing" + "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -361,6 +364,67 @@ func TestToEncryptionState(t *testing.T) { }, }, }, + + // scenario 11 + { + name: "kms write key", + input: encryptiontesting.CreateEncryptionCfgWithWriteKey([]encryptiontesting.EncryptionKeysResourceTuple{ + { + Resource: "secrets", + Keys: []apiserverconfigv1.Key{ + {Name: "1", Secret: "AAAAAAAAAAAAAAAAAAAAAA=="}, + }, + Modes: []string{"KMS"}, + }, + }), + output: map[schema.GroupResource]state.GroupResourceState{ + {Group: "", Resource: "secrets"}: { + WriteKey: state.KeyState{ + Key: apiserverconfigv1.Key{Name: "1", Secret: "AAAAAAAAAAAAAAAAAAAAAA=="}, + Mode: "KMS", + }, + ReadKeys: []state.KeyState{ + { + Key: apiserverconfigv1.Key{Name: "1", Secret: "AAAAAAAAAAAAAAAAAAAAAA=="}, + Mode: "KMS", + }, + }, + }, + }, + }, + + // scenario 12 + { + name: "kms write key and aescbc read key", + input: encryptiontesting.CreateEncryptionCfgWithWriteKey([]encryptiontesting.EncryptionKeysResourceTuple{ + { + Resource: "secrets", + Keys: []apiserverconfigv1.Key{ + {Name: "2", Secret: "AAAAAAAAAAAAAAAAAAAAAA=="}, + {Name: "1", Secret: "MTcxNTgyYTBmY2Q2YzVmZGI2NWNiZjVhM2U5MjQ5ZDc="}, // # notsecret + }, + Modes: []string{"KMS", "aescbc"}, + }, + }), + output: map[schema.GroupResource]state.GroupResourceState{ + {Group: "", Resource: "secrets"}: { + WriteKey: state.KeyState{ + Key: apiserverconfigv1.Key{Name: "2", Secret: "AAAAAAAAAAAAAAAAAAAAAA=="}, + Mode: "KMS", + }, + ReadKeys: []state.KeyState{ + { + Key: apiserverconfigv1.Key{Name: "2", Secret: "AAAAAAAAAAAAAAAAAAAAAA=="}, + Mode: "KMS", + }, + { + Key: apiserverconfigv1.Key{Name: "1", Secret: "MTcxNTgyYTBmY2Q2YzVmZGI2NWNiZjVhM2U5MjQ5ZDc="}, // # notsecret + Mode: "aescbc", + }, + }, + }, + }, + }, } for _, scenario := range scenarios { @@ -522,6 +586,44 @@ func TestFromEncryptionState(t *testing.T) { // scenario 6 // TODO: encryption on after being off + + // scenario 7 + { + name: "kms write key", + grs: []schema.GroupResource{{Group: "", Resource: "secrets"}}, + targetNs: "kms", + writeKeyIn: encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1), + makeOutput: func(writeKey *corev1.Secret, readKeys []*corev1.Secret) []apiserverconfigv1.ResourceConfiguration { + rs := apiserverconfigv1.ResourceConfiguration{} + rs.Resources = []string{"secrets"} + rs.Providers = []apiserverconfigv1.ProviderConfiguration{ + {KMS: keyToKMSConfiguration(writeKey, "secrets")}, + {Identity: &apiserverconfigv1.IdentityConfiguration{}}, + } + return []apiserverconfigv1.ResourceConfiguration{rs} + }, + }, + + // scenario 8 + { + name: "kms write key and aescbc read key", + grs: []schema.GroupResource{{Group: "", Resource: "secrets"}}, + targetNs: "kms", + writeKeyIn: encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 2), + readKeysIn: []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithRawKey("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1, []byte("61def964fb967f5d7c44a2af8dab6865")), + }, + makeOutput: func(writeKey *corev1.Secret, readKeys []*corev1.Secret) []apiserverconfigv1.ResourceConfiguration { + rs := apiserverconfigv1.ResourceConfiguration{} + rs.Resources = []string{"secrets"} + rs.Providers = []apiserverconfigv1.ProviderConfiguration{ + {KMS: keyToKMSConfiguration(writeKey, "secrets")}, + {AESCBC: keyToAESConfiguration(readKeys[0])}, + {Identity: &apiserverconfigv1.IdentityConfiguration{}}, + } + return []apiserverconfigv1.ResourceConfiguration{rs} + }, + }, } for _, scenario := range scenarios { @@ -556,8 +658,10 @@ func TestFromEncryptionState(t *testing.T) { actualOutput := FromEncryptionState(grState) expectedOutput := scenario.makeOutput(scenario.writeKeyIn, scenario.readKeysIn) - if !cmp.Equal(expectedOutput, actualOutput.Resources) { - t.Fatal(fmt.Errorf("%s", cmp.Diff(expectedOutput, actualOutput.Resources))) + // Ignore KMS Name field since it contains a random suffix + ignoreKMSName := cmpopts.IgnoreFields(apiserverconfigv1.KMSConfiguration{}, "Name") + if !cmp.Equal(expectedOutput, actualOutput.Resources, ignoreKMSName) { + t.Fatal(fmt.Errorf("%s", cmp.Diff(expectedOutput, actualOutput.Resources, ignoreKMSName))) } }) } @@ -578,6 +682,21 @@ func keyToAESConfiguration(key *corev1.Secret) *apiserverconfigv1.AESConfigurati } } +func keyToKMSConfiguration(key *corev1.Secret, resource string) *apiserverconfigv1.KMSConfiguration { + keyID, ok := state.NameToKeyID(key.Name) + if !ok { + panic(fmt.Sprintf("invalid test secret name %q", key.Name)) + } + return &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: fmt.Sprintf("%d-test%d", keyID, keyID), + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{ + Duration: 10 * time.Second, + }, + } +} + func newFakeIdentityEncodedKeyForTest() string { return "AAAAAAAAAAAAAAAAAAAAAA==" } diff --git a/pkg/operator/encryption/kms/helpers.go b/pkg/operator/encryption/kms/helpers.go index c5031ce2ab..0c469f7684 100644 --- a/pkg/operator/encryption/kms/helpers.go +++ b/pkg/operator/encryption/kms/helpers.go @@ -2,12 +2,18 @@ package kms import ( "fmt" + "time" "github.com/openshift/api/features" "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" corev1 "k8s.io/api/core/v1" ) +const ( + DefaultEndpoint = "unix:///var/run/kmsplugin/kms.sock" + DefaultTimeout = 10 * time.Second +) + // AddKMSPluginVolumeAndMountToPodSpec conditionally adds the KMS plugin volume mount to the specified container. // It assumes the pod spec does not already contain the KMS volume or mount; no deduplication is performed. // Deprecated: this is a temporary solution to get KMS TP v1 out. We should come up with a different approach afterwards. diff --git a/pkg/operator/encryption/preconditions_test.go b/pkg/operator/encryption/preconditions_test.go index afad0bb038..07eca5f4de 100644 --- a/pkg/operator/encryption/preconditions_test.go +++ b/pkg/operator/encryption/preconditions_test.go @@ -72,6 +72,21 @@ func TestEncryptionEnabledPrecondition(t *testing.T) { }, expectedPreconditionsToBeReady: true, }, + + // scenario 6 + { + name: "encryption on, currentMode set to KMS", + encryptionType: configv1.EncryptionTypeKMS, + expectedPreconditionsToBeReady: true, + }, + + // scenario 7 + { + name: "encryption off on previously KMS enabled cluster, with existing KMS key secret", + encryptionType: configv1.EncryptionTypeIdentity, + existingSecret: encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("oas", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1), + expectedPreconditionsToBeReady: true, + }, } for _, scenario := range scenarios { diff --git a/pkg/operator/encryption/secrets/secrets.go b/pkg/operator/encryption/secrets/secrets.go index 4e54317c7d..aa9daf3204 100644 --- a/pkg/operator/encryption/secrets/secrets.go +++ b/pkg/operator/encryption/secrets/secrets.go @@ -58,9 +58,17 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { key.ExternalReason = v } + if v, ok := s.Annotations[EncryptionSecretKMSConfig]; ok && len(v) > 0 { + kmsConfiguration := &apiserverconfigv1.KMSConfiguration{} + if err := json.Unmarshal([]byte(v), kmsConfiguration); err != nil { + return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid %s annotation: %v", s.Namespace, s.Name, EncryptionSecretKMSConfig, err) + } + key.KMSConfiguration = kmsConfiguration + } + keyMode := state.Mode(s.Annotations[encryptionSecretMode]) switch keyMode { - case state.AESCBC, state.AESGCM, state.SecretBox, state.Identity: + case state.AESCBC, state.AESGCM, state.SecretBox, state.Identity, state.KMS: key.Mode = keyMode default: return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid mode: %s", s.Namespace, s.Name, keyMode) @@ -113,6 +121,14 @@ func FromKeyState(component string, ks state.KeyState) (*corev1.Secret, error) { s.Annotations[EncryptionSecretMigratedResources] = string(bs) } + if ks.KMSConfiguration != nil { + ks, err := json.Marshal(ks.KMSConfiguration) + if err != nil { + return nil, err + } + s.Annotations[EncryptionSecretKMSConfig] = string(ks) + } + return s, nil } diff --git a/pkg/operator/encryption/secrets/secrets_test.go b/pkg/operator/encryption/secrets/secrets_test.go index 81b81b017b..c4ca9f1f2a 100644 --- a/pkg/operator/encryption/secrets/secrets_test.go +++ b/pkg/operator/encryption/secrets/secrets_test.go @@ -16,6 +16,8 @@ import ( func TestRoundtrip(t *testing.T) { now, _ := time.Parse(time.RFC3339, time.Now().Format(time.RFC3339)) + emptyKey := make([]byte, 16) + tests := []struct { name string component string @@ -111,6 +113,50 @@ func TestRoundtrip(t *testing.T) { ExternalReason: "external", }, }, + { + name: "full kms", + component: "kms", + ks: state.KeyState{ + Key: v1.Key{ + Name: "1", + Secret: base64.StdEncoding.EncodeToString(emptyKey), + }, + Backed: true, // this will be set by ToKeyState() + Mode: "KMS", + KMSConfiguration: &v1.KMSConfiguration{ + APIVersion: "v2", + Name: "kms-plugin", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + }, + Migrated: state.MigrationState{ + Timestamp: now, + Resources: []schema.GroupResource{ + {Resource: "secrets"}, + {Resource: "configmaps"}, + {Group: "networking.openshift.io", Resource: "routes"}, + }, + }, + InternalReason: "internal", + ExternalReason: "external", + }, + }, + { + name: "sparse kms", + component: "kms", + ks: state.KeyState{ + Key: v1.Key{ + Name: "2", + Secret: base64.StdEncoding.EncodeToString(emptyKey), + }, + Backed: true, // this will be set by ToKeyState() + Mode: "KMS", + KMSConfiguration: &v1.KMSConfiguration{ + APIVersion: "v2", + Name: "kms-plugin", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/operator/encryption/secrets/types.go b/pkg/operator/encryption/secrets/types.go index 7161e4a124..443c7975e1 100644 --- a/pkg/operator/encryption/secrets/types.go +++ b/pkg/operator/encryption/secrets/types.go @@ -49,6 +49,9 @@ const ( // by the encryption controllers. Its sole purpose is to prevent the accidental // deletion of secrets by enforcing a two phase delete. EncryptionSecretFinalizer = "encryption.apiserver.operator.openshift.io/deletion-protection" + + // EncryptionSecretKMSConfig is the annotation that stores the encoded KMS configuration. + EncryptionSecretKMSConfig = "encryption.apiserver.operator.openshift.io/kms-config" ) // MigratedGroupResources is the data structured stored in the diff --git a/pkg/operator/encryption/state/types.go b/pkg/operator/encryption/state/types.go index 460c21bfa2..61313d745e 100644 --- a/pkg/operator/encryption/state/types.go +++ b/pkg/operator/encryption/state/types.go @@ -40,6 +40,8 @@ type KeyState struct { InternalReason string // the user via unsupportConfigOverrides.encryption.reason triggered this key. ExternalReason string + // Encoded KMSConfiguration that stores the KMS related fields + KMSConfiguration *apiserverconfigv1.KMSConfiguration } type MigrationState struct { @@ -60,6 +62,7 @@ const ( AESGCM Mode = "aesgcm" SecretBox Mode = "secretbox" // available from the first release, see defaultMode below Identity Mode = "identity" // available from the first release, see defaultMode below + KMS Mode = "KMS" // only supports KMS v2 // Changing this value requires caution to not break downgrades. // Specifically, if some new Mode is released in version X, that new Mode cannot diff --git a/pkg/operator/encryption/statemachine/transition_test.go b/pkg/operator/encryption/statemachine/transition_test.go index 774c4789cd..b4a4253ca7 100644 --- a/pkg/operator/encryption/statemachine/transition_test.go +++ b/pkg/operator/encryption/statemachine/transition_test.go @@ -985,6 +985,305 @@ func TestGetDesiredEncryptionState(t *testing.T) { }, }), }, + { + "no config, KMS secret exists => first config is created with KMS", + args{ + nil, + "kms", + []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 1), + }, + []schema.GroupResource{{Group: "", Resource: "secrets"}}, + }, + equalsConfig(&apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }}, + }, + }}), + }, + { + "config exists with AESCBC, KMS secret added => KMS added as read key (migration scenario)", + args{ + &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "1", + Secret: base64.StdEncoding.EncodeToString([]byte("11ea7c91419a68fd1224f88d50316b4e")), + }}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + "kms", + []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithRawKey("kms", nil, 1, []byte("11ea7c91419a68fd1224f88d50316b4e")), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 2), + }, + []schema.GroupResource{{Group: "", Resource: "secrets"}}, + }, + equalsConfig(&apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "1", + Secret: base64.StdEncoding.EncodeToString([]byte("11ea7c91419a68fd1224f88d50316b4e")), + }}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }, + }, + }), + }, + { + "config exists with KMS, read keys are consistent => new write key is set", + args{ + &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + "kms", + []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 1), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 2), + }, + []schema.GroupResource{{Group: "", Resource: "secrets"}}, + }, + equalsConfig(&apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }, + }, + }), + }, + { + "config exists with KMS, read+write keys consistent, not migrated => nothing changes", + args{ + &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + "kms", + []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 1), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", nil, 2), + }, + []schema.GroupResource{{Group: "", Resource: "secrets"}}, + }, + equalsConfig(&apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }), + }, + { + "KMS has converged after migrating from AESCBC => nothing changes", + args{ + &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "1", + Secret: base64.StdEncoding.EncodeToString([]byte("21ea7c91419a68fd1224f88d50316b4e")), + }}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + "kms", + []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithRawKey("kms", nil, 1, []byte("21ea7c91419a68fd1224f88d50316b4e")), + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 2), + }, + []schema.GroupResource{{Group: "", Resource: "secrets"}}, + }, + equalsConfig(&apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "2-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "1", + Secret: base64.StdEncoding.EncodeToString([]byte("21ea7c91419a68fd1224f88d50316b4e")), + }}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }), + }, + { + "AESCBC has converged after migrating back from KMS => nothing changes", + args{ + &apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "2", + Secret: base64.StdEncoding.EncodeToString([]byte("21ea7c91419a68fd1224f88d50316b4e")), + }}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + "kms", + []*corev1.Secret{ + encryptiontesting.CreateEncryptionKeySecretWithKMSConfig("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 1), + encryptiontesting.CreateEncryptionKeySecretWithRawKey("kms", nil, 2, []byte("21ea7c91419a68fd1224f88d50316b4e")), + }, + []schema.GroupResource{{Group: "", Resource: "secrets"}}, + }, + equalsConfig(&apiserverconfigv1.EncryptionConfiguration{ + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: []apiserverconfigv1.Key{{ + Name: "2", + Secret: base64.StdEncoding.EncodeToString([]byte("21ea7c91419a68fd1224f88d50316b4e")), + }}, + }, + }, { + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1-secrets", + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/operator/encryption/testing/helpers.go b/pkg/operator/encryption/testing/helpers.go index d4a2cd5ad1..a802b91980 100644 --- a/pkg/operator/encryption/testing/helpers.go +++ b/pkg/operator/encryption/testing/helpers.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/openshift/library-go/pkg/operator/encryption/kms" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,6 +25,7 @@ const ( encryptionSecretKeyDataForTest = "encryption.apiserver.operator.openshift.io-key" encryptionSecretMigratedTimestampForTest = "encryption.apiserver.operator.openshift.io/migrated-timestamp" encryptionSecretMigratedResourcesForTest = "encryption.apiserver.operator.openshift.io/migrated-resources" + encryptionSecretKMSConfigForTest = "encryption.apiserver.operator.openshift.io/kms-config" ) func CreateEncryptionKeySecretNoData(targetNS string, grs []schema.GroupResource, keyID uint64) *corev1.Secret { @@ -94,6 +96,30 @@ func CreateExpiredMigratedEncryptionKeySecretWithRawKey(targetNS string, grs []s return CreateMigratedEncryptionKeySecretWithRawKey(targetNS, grs, keyID, rawKey, time.Now().Add(-(time.Hour*24*7 + time.Hour))) } +func CreateEncryptionKeySecretWithKMSConfig(targetNS string, grs []schema.GroupResource, keyID uint64) *corev1.Secret { + emptyKey := make([]byte, 16) + secret := CreateEncryptionKeySecretWithRawKeyWithMode(targetNS, grs, keyID, emptyKey, "KMS") + kmsConfig := &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: fmt.Sprintf("%d", keyID), + Endpoint: kms.DefaultEndpoint, + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + } + kmsConfigJSON, _ := json.Marshal(kmsConfig) + secret.Annotations[encryptionSecretKMSConfigForTest] = string(kmsConfigJSON) + return secret +} + +func CreateMigratedEncryptionKeySecretWithKMSConfig(targetNS string, grs []schema.GroupResource, keyID uint64, ts time.Time) *corev1.Secret { + secret := CreateEncryptionKeySecretWithKMSConfig(targetNS, grs, keyID) + secret.Annotations[encryptionSecretMigratedTimestampForTest] = ts.Format(time.RFC3339) + return secret +} + +func CreateExpiredMigratedEncryptionKeySecretWithKMSConfig(targetNS string, grs []schema.GroupResource, keyID uint64) *corev1.Secret { + return CreateMigratedEncryptionKeySecretWithKMSConfig(targetNS, grs, keyID, time.Now().Add(-(time.Hour*24*7 + time.Hour))) +} + func CreateDummyKubeAPIPod(name, namespace string, nodeName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -243,6 +269,17 @@ func createProviderCfg(mode string, key apiserverconfigv1.Key) *apiserverconfigv return &apiserverconfigv1.ProviderConfiguration{ Identity: &apiserverconfigv1.IdentityConfiguration{}, } + case "KMS": + // key.Name contains the key ID + // Use a deterministic suffix for testing + return &apiserverconfigv1.ProviderConfiguration{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: fmt.Sprintf("%s-test%s", key.Name, key.Name), + Endpoint: "unix:///var/run/kmsplugin/kms.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + } default: return &apiserverconfigv1.ProviderConfiguration{ AESCBC: &apiserverconfigv1.AESConfiguration{ diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index 877d9bdced..c99680ee7a 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -411,6 +411,100 @@ func TestEncryptionIntegration(tt *testing.T) { "kubeapiservers.operator.openshift.io=aescbc:7,aescbc:5,identity;kubeschedulers.operator.openshift.io=aescbc:7,aescbc:5,identity", ) waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Switch to KMS") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS"}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(7) + kms8 := kmsProviderName("kubeapiservers", "8") + kms8Sched := kmsProviderName("kubeschedulers", "8") + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:7,kms:%s,aescbc:5,identity;kubeschedulers.operator.openshift.io=aescbc:7,kms:%s,aescbc:5,identity", kms8, kms8Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:7,aescbc:5,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:7,aescbc:5,identity", kms8, kms8Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:7,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:7,identity", kms8, kms8Sched), + ) + waitForMigration("8") + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Switch back to aescbc from KMS") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"aescbc"}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(8) + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:9,aescbc:7,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:9,aescbc:7,identity", kms8, kms8Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:9,kms:%s,aescbc:7,identity;kubeschedulers.operator.openshift.io=aescbc:9,kms:%s,aescbc:7,identity", kms8, kms8Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:9,kms:%s,identity;kubeschedulers.operator.openshift.io=aescbc:9,kms:%s,identity", kms8, kms8Sched), + ) + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Switch back to KMS") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS"}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(9) + kms10 := kmsProviderName("kubeapiservers", "10") + kms10Sched := kmsProviderName("kubeschedulers", "10") + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:9,kms:%s,kms:%s,identity;kubeschedulers.operator.openshift.io=aescbc:9,kms:%s,kms:%s,identity", kms10, kms8, kms10Sched, kms8Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:9,kms:%s,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:9,kms:%s,identity", kms10, kms8, kms10Sched, kms8Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:9,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:9,identity", kms10, kms10Sched), + ) + waitForMigration("10") + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Rotate KMS key via aescbc (KMS->AESCBC->KMS)") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"aescbc"}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(10) + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,aescbc:9,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,aescbc:9,identity", kms10, kms10Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:11,kms:%s,aescbc:9,identity;kubeschedulers.operator.openshift.io=aescbc:11,kms:%s,aescbc:9,identity", kms10, kms10Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:11,kms:%s,identity;kubeschedulers.operator.openshift.io=aescbc:11,kms:%s,identity", kms10, kms10Sched), + ) + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Switch back to KMS after rotation") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS"}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(11) + kms12 := kmsProviderName("kubeapiservers", "12") + kms12Sched := kmsProviderName("kubeschedulers", "12") + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=aescbc:11,kms:%s,kms:%s,identity;kubeschedulers.operator.openshift.io=aescbc:11,kms:%s,kms:%s,identity", kms12, kms10, kms12Sched, kms10Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,kms:%s,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,kms:%s,identity", kms12, kms10, kms12Sched, kms10Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity", kms12, kms12Sched), + ) + waitForMigration("12") + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Delete the encryption-config while in KMS mode") + _, err = kubeClient.CoreV1().Secrets("openshift-config-managed").Patch(ctx, fmt.Sprintf("encryption-config-%s", component), types.JSONPatchType, []byte(`[{"op":"remove","path":"/metadata/finalizers"}]`), metav1.PatchOptions{}) + require.NoError(t, err) + err = kubeClient.CoreV1().Secrets("openshift-config-managed").Delete(ctx, fmt.Sprintf("encryption-config-%s", component), metav1.DeleteOptions{}) + require.NoError(t, err) + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity", kms12, kms12Sched), + ) + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Delete the operand config while in KMS mode") + deployer.DeleteOperandConfig() + waitForConfigs( + // kms12 is migrated and hence only one needed, but we rotate through identity + fmt.Sprintf("kubeapiservers.operator.openshift.io=identity,kms:%s,aescbc:11;kubeschedulers.operator.openshift.io=identity,kms:%s,aescbc:11", kms12, kms12Sched), + // kms12 is migrated, plus one backed key (11) + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity", kms12, kms12Sched), + ) + waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + + t.Logf("Switch to identity from KMS") + _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"identity"}}}`), metav1.PatchOptions{}) + require.NoError(t, err) + waitForKeys(12) + waitForConfigs( + fmt.Sprintf("kubeapiservers.operator.openshift.io=kms:%s,aescbc:11,identity,aesgcm:13;kubeschedulers.operator.openshift.io=kms:%s,aescbc:11,identity,aesgcm:13", kms12, kms12Sched), + fmt.Sprintf("kubeapiservers.operator.openshift.io=identity,kms:%s,aescbc:11,aesgcm:13;kubeschedulers.operator.openshift.io=identity,kms:%s,aescbc:11,aesgcm:13", kms12, kms12Sched), + ) + waitForConditionStatus("Encrypted", operatorv1.ConditionFalse) } const encryptionTestOperatorCRD = ` @@ -495,6 +589,8 @@ func toString(c *apiserverv1.EncryptionConfiguration) string { s = "aescbc:" + p.AESCBC.Keys[0].Name case p.AESGCM != nil: s = "aesgcm:" + p.AESGCM.Keys[0].Name + case p.KMS != nil: + s = "kms:" + p.KMS.Name case p.Identity != nil: s = "identity" } @@ -719,3 +815,9 @@ func (p *provider) EncryptedGRs() []schema.GroupResource { func (p *provider) ShouldRunEncryptionControllers() (bool, error) { return true, nil } + +// kmsProviderName computes the KMS provider name with checksum for a given resource and key ID. +// Format: {keyID}-{resource} +func kmsProviderName(resource, keyID string) string { + return fmt.Sprintf("%s-%s", keyID, resource) +}