diff --git a/examples/psmdb/Makefile b/examples/psmdb/Makefile index 03c92b0..4f6ffb4 100644 --- a/examples/psmdb/Makefile +++ b/examples/psmdb/Makefile @@ -22,9 +22,12 @@ generate-openapi: openapi-gen ## Generate OpenAPI definitions for custom spec ty --go-header-file ../../hack/boilerplate.go.txt \ github.com/openeverest/provider-sdk/examples/psmdb/types +## Optional test directory name for running subset of integration tests +TEST ?= + .PHONY: test-integration -test-integration: ## Run integration tests against K8S cluster - . ./test/vars.sh && kubectl kuttl test --config ./test/integration/kuttl.yaml +test-integration: ## Run integration tests. Usage: make test-integration TEST= + . ./test/vars.sh && kubectl kuttl test --config ./test/integration/kuttl.yaml $(if $(TEST),--test $(TEST)) .PHONY: openapi-gen openapi-gen: $(OPENAPI_GEN) ## Download openapi-gen locally if necessary diff --git a/examples/psmdb/README.md b/examples/psmdb/README.md index 9cc56c2..3d9ce6e 100644 --- a/examples/psmdb/README.md +++ b/examples/psmdb/README.md @@ -35,7 +35,7 @@ examples/psmdb/ 3. Install the PSMDB operator: ```bash - kubectl apply --server-side -f https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v1.21.1/deploy/bundle.yaml + kubectl apply --server-side -f https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v1.21.1/deploy/cw-bundle.yaml ``` **Note:** This is a PoC requirement. In production, the underlying database operator (PSMDB in this case) should be packaged within the provider's Helm chart to ensure it installs automatically with the provider. diff --git a/examples/psmdb/datastore-split-horizon.yaml b/examples/psmdb/datastore-split-horizon.yaml new file mode 100644 index 0000000..a4a87f5 --- /dev/null +++ b/examples/psmdb/datastore-split-horizon.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: split-horizon-config + namespace: default +data: + baseDomainNameSuffix: mydomain.com + secretName: split-horizon-secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: split-horizon-secret + namespace: default +type: Opaque +data: + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmJZczVCa2xqSHRKckZkOFNnc0xhNzZvQ1FVQWdTcU1OdkZKVWxMSjBtMlM4eDI1ClZHWGFPWDM5Q1hHKzIxTzg5VGRoWHRRSU5nMEoxbTdmSlVFMDVGS2xKVTBPUWxRV2hETDNKWFF6UUdJc0FZV1YKV3Rib1VybjJvZVNoWHJmYkhOR3J4YnlmRitkcEJPRlpLY0pwU3Nta0ROMjVIVjlXbHdqdHFEVi9qZW92UHMvbgpPUm9VS0RJRjlSRUVwemtkbVNSbjlBNlBPb1A1UlJ6VkYzWTNlQ3JEK0RRbnQ3Qm5ndEtnWGZqbUg0RDJtSHZmCnBtWWJ0T3pjRzhNRXpBNGZaMldib2o3QlUxYTdZUmxiOVBGN1N1b1hBdldRMUJxNHlmWDN2em1KcjQ5RWZtRWoKbW4xZjdSeFdqV2M2QmxXaUFsVzFqeWJOc2VoZXl3MS9LSjk3V1FJREFRQUJBb0lCQUJZVzI1YUNGcEVMVEc1aQpuK1JUc2FSZHBsMXBmWS9zb3plbEErVnY0aFBNUWlWb2J6dEVZa0xhT0grMFpMV1BSQ1BWeTFMQ004UU9ZOWc1CjMrWHJ1Q1BET3pzekJBOWxVTnRhLzFPM09sMzdhRllUaHFyVmRlYm5CQkNBNldpNDFleWV2Y1pZQW5yRExVTmUKRlZhVFRsVUEwa2NVdkpDTzJLdGNwT1oyZHpnUDZpQ3BiUmNPUGVieDdHZXNiS2ZaRkxSOUNUSlgvUXY0YUFvMQpvL3BLYTZEeXk4V0MzNWtxOWQ3cTk4dmV1T3VMSWxPZ0k3UGdYMVBxUHJIOCs1OEI2YUduaC9Oa1VWMkd6OExKCkNSQmZoZW5RQ2EyVkM5UU9qLzhsbEVWdERTMXlNUEFUcG14ZWRESjI4ODNtWjRxNllaUjM3T0VVTVdUSWtzUWgKY0RNZUNlRUNnWUVBL0FSbldZUzFhYjQxeDVtcVlmbnhXMktvQmxvTm5LbWZGa1IvZEVqWE1ReEc3eFFSSTBNTAovenRHcUxxUHlaRGVNVGQ3ZzZVOHVmWWVPMURMQ3RjK3JwRlFVSWt1aHdxUEpVRVNiRUhMYzcvSkF5czREaUcvClVPWEJRd21RQmdlazVFaW9IaHhCNWQyWE4xTk5iemFnOFZsZkF4WDhlQnhFejFZa2RFZ0NQT0VDZ1lFQXdMV3oKRFFMaFNndW5pQjhZaXFmalQyaldZREF1NHY0ays5L1Izc1BkTm9xOVpCRk16VWE3N25pWFFOTUpFaVg5V2I2SQp2ZDBBdTAzdzdFYTNmbDIycWtmWHljQ1ZsYWxXVXJ5VjFyamxHS013VytEeEVHbUpYRWtEek1ycnRINU8xS0VCCmpFRWtPTC92Wkl6Z245UHNRSHNXTWdTbXFONDNmUCtjamdLYVZYa0NnWUJMRWZ5L083cldidVNTT29INGdYMlYKM1VOejhPdFJHVzNjTWpkTktrMS92TXA4ZVJ2Snh6VVJxRlNaK2tqT29DcXZ1bmsrYzhBdEhOVlFrZmFKbWloLwowNlY2K0FJMkU3MGtPY2dGRzJ4QlpJVzZQZXVLdWg3Rk9FdGpicnZLTUFpOFA3QmtsOEpCZU1xTW5uSFlpUXRVCkdXMGwvQ3lpa3Jra2tlSjJDT1V4d1FLQmdFUVVyUkh1cjRyS1BVQ1F3OG5RY0RUUXM5TzlrZ0x0aUVGWG5EeFgKOCtIZDkvVFBTOVBGcG9Va0kwTnFpdXpYY1A3d21qeUJSRTNueGpLaTlSWjJveDdiVExmaENyZVo0SDVRVTV0TgpMTnFjWkd4Qk5zajJqK21EZmcwdXIwRFAwcWU1emVNdjdFMEVPZDNMQzF4THNVNUNiZC96MXJFWCtJQjNpV1orCk11bjVBb0dCQUsyV3E0dFlOeWxQZVpiK2RiWWRUSmlPTkswYVdmZEs1WkpxSXdmcXNhb2xPdUIyVnhvVDVoWGkKQml4YWYwWFlPL3JlVUd1SjR5dXNQRzNOZEdEZ1h5cHFoWFU3Z2xRSmQ4SUMxMDh5elVPQjNOZy8xRGtkRjYrdwpPUWdLdmpvUXVaWDc0bkdUWG1lL3ZDSVhQRjB3Sy9IaFhhemZyWlIzMjc2VDkwQkUzdndECi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lVWTlLTWtlTC82RVhQaitWTjQ0N0hvWUY1TlpVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lqRU9NQXdHQTFVRUNoTUZVRk5OUkVJeEVEQU9CZ05WQkFNVEIxSnZiM1FnUTBFd0hoY05NalV4TVRBMQpNRGMxTmpBd1doY05NekF4TVRBME1EYzFOakF3V2pBaU1RNHdEQVlEVlFRS0V3VlFVMDFFUWpFUU1BNEdBMVVFCkF4TUhVbTl2ZENCRFFUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyMkxPUVoKSll4N1NheFhmRW9MQzJ1K3FBa0ZBSUVxakRieFNWSlN5ZEp0a3ZNZHVWUmwyamw5L1FseHZ0dFR2UFUzWVY3VQpDRFlOQ2RadTN5VkJOT1JTcFNWTkRrSlVGb1F5OXlWME0wQmlMQUdGbFZyVzZGSzU5cUhrb1Y2MzJ4elJxOFc4Cm54Zm5hUVRoV1NuQ2FVckpwQXpkdVIxZlZwY0k3YWcxZjQzcUx6N1A1emthRkNneUJmVVJCS2M1SFpra1ovUU8KanpxRCtVVWMxUmQyTjNncXcvZzBKN2V3WjRMU29GMzQ1aCtBOXBoNzM2Wm1HN1RzM0J2REJNd09IMmRsbTZJKwp3Vk5XdTJFWlcvVHhlMHJxRndMMWtOUWF1TW4xOTc4NWlhK1BSSDVoSTVwOVgrMGNWbzFuT2daVm9nSlZ0WThtCnpiSG9Yc3NOZnlpZmUxa0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdFR01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRzVXeWtBb2d0OG1hNzRTeXRxeXhxTS9TMm1NTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ2pmeU5oaDF2cjN3MVZXalRLajl4bUNrMU9YK1pBcVI3NkpkWkk0UHhnQ1hRQkNid1d6enBLCmtmeEFTTUowNE5LYnJFZ0tnWXczNG96clNvbVlMZndETm82czlLenBBZEQ2Z2F3Q0dPZXEzNzM4T3BSSU44YTkKNENZM0ZMRCs4emExcVI0ZkVtV1dOdnFvNlBNeDE3VGVjK092WnhJcTRXakZlRDRnZ3pYeWhqMTZzSGE2Z2YyYQphbjZvbktVVnhsYTdtTGdhY2taRzNXOTdOMlNBcXR3U252d3luYXNvaWFSeEM4OTRweC9FbnpNeXcwMElqeEFkCjZGMndWYVpqWmhBRDlRMkdIV29iS0RWK05LUDYxaXVWQjA4NVJZWWxVTXBxd3J4bkhBWllJQjk4SXJmVjFyRlUKQUhQOWRFNnVqMnFvTDQyWE5rN0ZvamJJRmw4RWIyRXQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + +--- +# Example 1: DataStore with Split Horizon DNS Configuration at Component Level +apiVersion: everest.percona.com/v2alpha1 +kind: DataStore +metadata: + name: psmdb-split-horizon + namespace: default +spec: + provider: psmdb + components: + engine: + type: mongod + version: 8.0.8-3 + replicas: 3 + storage: + size: 10Gi + customSpec: + splitHorizonDNSRef: + name: split-horizon-config + backupAgent: + type: backup + replicas: 1 + proxy: + type: mongos + version: 8.0.8-3 + replicas: 3 + # service: + # exposeType: ExternalName diff --git a/examples/psmdb/internal/provider.go b/examples/psmdb/internal/provider.go index 87e5e7c..169fee2 100644 --- a/examples/psmdb/internal/provider.go +++ b/examples/psmdb/internal/provider.go @@ -47,6 +47,10 @@ const ( defaultBackupStartingTimeout = 120 ) +const ( + topologySharded = "sharded" +) + var maxUnavailable = intstr.FromInt(1) func defaultSpec() psmdbv1.PerconaServerMongoDBSpec { @@ -163,7 +167,7 @@ func configureReplsets(c *sdk.Context) []*psmdbv1.ReplsetSpec { engine := spec.Components[ComponentEngine] // TODO: implement disabling - if spec.Topology == nil || spec.Topology.Type != "sharded" { + if spec.Topology == nil || spec.Topology.Type != topologySharded { return []*psmdbv1.ReplsetSpec{ configureReplset(rsName(0), engine.Replicas, engine.Resources, engine.Storage, true), } @@ -191,7 +195,7 @@ func configureConfigServerReplset(c *sdk.Context) *psmdbv1.ReplsetSpec { cfgSrv := spec.Components[ComponentConfigServer] // TODO: implement disabling - if spec.Topology == nil || spec.Topology.Type != "sharded" { + if spec.Topology == nil || spec.Topology.Type != topologySharded { return replset } @@ -292,7 +296,7 @@ func SyncPSMDB(c *sdk.Context) error { psmdb.Spec.ImagePullPolicy = corev1.PullIfNotPresent psmdb.Spec.Replsets = configureReplsets(c) - if c.DB().Spec.Topology != nil && c.DB().Spec.Topology.Type == "sharded" { + if c.DB().Spec.Topology != nil && c.DB().Spec.Topology.Type == topologySharded { psmdb.Spec.Sharding.Enabled = true psmdb.Spec.Sharding.ConfigsvrReplSet = configureConfigServerReplset(c) psmdb.Spec.Sharding.Mongos = configureMongos(c) @@ -306,6 +310,17 @@ func SyncPSMDB(c *sdk.Context) error { SSLInternal: c.Name() + "-ssl-internal", } + splitHorizonRef, err := getSpritHorizonFromCustomSpec(c) + if err != nil { + return fmt.Errorf("failed to retrieve split horizon ref: %w", err) + } + + if splitHorizonRef != nil { + if err := configureSplitHorizon(c, psmdb, splitHorizonRef); err != nil { + return fmt.Errorf("failed to apply split horizon configuration to PSMDB: %w", err) + } + } + if err := c.Apply(psmdb); err != nil { return err } @@ -326,6 +341,19 @@ func StatusPSMDB(c *sdk.Context) (sdk.Status, error) { } switch psmdb.Status.State { case psmdbv1.AppStateReady: + // Check if split horizon DNS is configured and ready + splitHorizonRef, err := getSpritHorizonFromCustomSpec(c) + if err != nil { + return sdk.Failed(err.Error()), nil + } + + if splitHorizonRef != nil { + // Split horizon is configured, check if it's ready + if err := statusSplitHorizon(c, psmdb); err != nil { + return sdk.Creating("Waiting for split horizon DNS to be ready"), nil + } + } + return sdk.Running(), nil case psmdbv1.AppStateError: return sdk.Failed(psmdb.Status.Message), nil @@ -406,7 +434,7 @@ func PSMDBMetadata() *sdk.ProviderMetadata { // backup is the backup agent component ComponentTypeBackup: { Versions: []sdk.ComponentVersionMeta{ - {Version: "2.9.1", Image: "percona/percona-server-mongodb-backup:2.9.1", Default: true}, + {Version: "2.9.1", Image: "percona/percona-backup-mongodb:2.9.1", Default: true}, }, }, // pmm is the Percona Monitoring and Management component diff --git a/examples/psmdb/internal/splithorizon.go b/examples/psmdb/internal/splithorizon.go new file mode 100644 index 0000000..72aabb7 --- /dev/null +++ b/examples/psmdb/internal/splithorizon.go @@ -0,0 +1,346 @@ +package provider + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "time" + + types "github.com/openeverest/provider-sdk/examples/psmdb/types" + sdk "github.com/openeverest/provider-sdk/pkg/controller" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" +) + +const ( + SplitHorizonConfigMapKeyBaseDomainNameSuffix = "baseDomainNameSuffix" + SplitHorizonConfigMapKeySecretName = "secretName" + CACertificateKey = "ca.crt" + CAKeyKey = "ca.key" +) + +const ( + splitHorizonExternalKey = "external" + publicIPPendingValue = "pending" +) + +var ( + errShardingNotSupported = errors.New("sharding is not supported for SplitHorizon DNS feature") + validityNotAfter = time.Date(9999, 12, 31, 23, 59, 59, 0, time.UTC) +) + +// getSpritHorizonFromCustomSpec retrieves the SplitHorizonDNSRef from the mongod component's CustomSpec. +// If SplitHorizonDNSRef is not set, it returns nil. +func getSpritHorizonFromCustomSpec(c *sdk.Context) (*types.SplitHorizonDNSRef, error) { + engine, ok := c.DB().Spec.Components[ComponentEngine] + if !ok { + return nil, nil + } + + if engine.CustomSpec == nil || engine.CustomSpec.Raw == nil { + return nil, nil + } + + mongodSpec := &types.MongodCustomSpec{} + if err := c.DecodeComponentCustomSpec(engine, mongodSpec); err != nil { + return nil, fmt.Errorf("failed to decode mongod CustomSpec: %w", err) + } + + return mongodSpec.SplitHorizonDNSRef, nil +} + +// getSplitHorizonConfig retrieves a pre-configured split horizon DNS configuration by reference. +// The configuration is expected to be stored as a ConfigMap in the specified or current namespace. +func getSplitHorizonConfig(c *sdk.Context, ref *types.SplitHorizonDNSRef) (*types.SplitHorizonDNSConfig, error) { + // Determine the namespace to look in + configNamespace := ref.Namespace + if configNamespace == "" { + // Use the DataStore's namespace + configNamespace = c.Namespace() + } + + // Retrieve the ConfigMap + configMapName := ref.Name + configMap := &corev1.ConfigMap{} + + if err := c.Get(configMap, configMapName); err != nil { + if k8serrors.IsNotFound(err) { + // ConfigMap not found, wait for it to be created + return nil, sdk.WaitFor(fmt.Sprintf("split horizon configuration ConfigMap %s not found in namespace %s", configMapName, configNamespace)) + } + return nil, fmt.Errorf("failed to retrieve split horizon configuration ConfigMap %s in namespace %s: %w", configMapName, configNamespace, err) + } + + // Parse the configuration from the ConfigMap + if configMap.Data == nil { + return nil, fmt.Errorf("split horizon ConfigMap has no data") + } + + baseDomain, ok := configMap.Data[SplitHorizonConfigMapKeyBaseDomainNameSuffix] + if !ok || baseDomain == "" { + return nil, fmt.Errorf("split horizon ConfigMap missing required key: baseDomainNameSuffix") + } + + secretName, ok := configMap.Data[SplitHorizonConfigMapKeySecretName] + if !ok || secretName == "" { + return nil, fmt.Errorf("split horizon ConfigMap missing required key: secretName") + } + + return &types.SplitHorizonDNSConfig{ + BaseDomainNameSuffix: baseDomain, + SecretName: secretName, + }, nil +} + +// configureSplitHorizon applies split horizon DNS configuration to PSMDB cluster spec. +// This configures the cluster for split horizon DNS setup, which allows clients to connect +// to MongoDB through different DNS names depending on the network/region they are in. +func configureSplitHorizon(c *sdk.Context, psmdb *psmdbv1.PerconaServerMongoDB, ref *types.SplitHorizonDNSRef) error { //nolint:funcorder + shdc, err := getSplitHorizonConfig(c, ref) + if err != nil { + return err + } + + spec := c.DB().Spec + engine := spec.Components[ComponentEngine] + + if spec.Topology != nil && spec.Topology.Type == topologySharded { + // For the time being SplitHorizon DNS feature can be applied for unsharded PSMDB clusters only. + // Later we may consider adding support for sharded clusters as well. + return errShardingNotSupported + } + + shdcCaSecret := new(corev1.Secret) + if err := c.Get(shdcCaSecret, shdc.SecretName); err != nil { + return err + } + + var nReplicas int32 + if engine.Replicas != nil { + nReplicas = *engine.Replicas + } + + horSpec := psmdbv1.HorizonsSpec{} + for i := range nReplicas { + horSpec[fmt.Sprintf("%s-rs0-%d", psmdb.GetName(), i)] = map[string]string{ + splitHorizonExternalKey: fmt.Sprintf("%s-rs0-%d-%s.%s", + psmdb.GetName(), + i, + psmdb.GetNamespace(), + shdc.BaseDomainNameSuffix), + } + } + psmdb.Spec.Replsets[0].Horizons = horSpec + + needCreateSecret := false // do not generate server certificate on each reconciliation loop + psmdbSplitHorizonSecretName := getSplitHorizonDNSConfigSecretName(psmdb.GetName()) + + psmdbSplitHorizonSecret := &corev1.Secret{} + if err := c.Get(psmdbSplitHorizonSecret, psmdbSplitHorizonSecretName); err != nil { + if k8serrors.IsNotFound(err) { + needCreateSecret = true + } else { + return err + } + } + + // TODO Should ConfigMap for Split Horizon DNS also hold certificates optionally just like CRD? + if needCreateSecret { + // Generate server TLS certificate for SplitHorizon DNS domains + psmdbSplitHorizonDomains := []string{ + // Internal SANs + "localhost", + psmdb.GetName() + "-rs0", + fmt.Sprintf("*.%s-rs0", psmdb.GetName()), + fmt.Sprintf("%s-rs0.%s", psmdb.GetName(), psmdb.GetNamespace()), + fmt.Sprintf("*.%s-rs0.%s", psmdb.GetName(), psmdb.GetNamespace()), + fmt.Sprintf("%s-rs0.%s.svc.cluster.local", psmdb.GetName(), psmdb.GetNamespace()), + fmt.Sprintf("*.%s-rs0.%s.svc.cluster.local", psmdb.GetName(), psmdb.GetNamespace()), + // External SANs + "*." + shdc.BaseDomainNameSuffix, + } + psmdbSplitHorizonServerCertBytes, psmdbSplitHorizonServerPrivKeyBytes, err := issueSplitHorizonCertificate( + shdcCaSecret.Data["ca.crt"], + shdcCaSecret.Data["ca.key"], + psmdbSplitHorizonDomains) + if err != nil { + return fmt.Errorf("issue split-horizon server TLS certificate: %w", err) + } + + // Store generated server TLS certificate in a secret. It is DB specific. + psmdbSplitHorizonSecret.SetName(psmdbSplitHorizonSecretName) + psmdbSplitHorizonSecret.SetNamespace(psmdb.GetNamespace()) + + // TODO: use controllerutil.CreateOrUpdate to update only if data is changed + psmdbSplitHorizonSecret.Data = map[string][]byte{ + "tls.crt": []byte(psmdbSplitHorizonServerCertBytes), + "tls.key": []byte(psmdbSplitHorizonServerPrivKeyBytes), + "ca.crt": shdcCaSecret.Data["ca.crt"], + } + psmdbSplitHorizonSecret.Type = corev1.SecretTypeTLS + + if err := c.Apply(psmdbSplitHorizonSecret); err != nil { + return fmt.Errorf("failed to create server TLS certificate secret: %w", err) + } + } + + // set reference to secret with certificate for external domain + if psmdb.Spec.Secrets == nil { + psmdb.Spec.Secrets = &psmdbv1.SecretsSpec{} + } + psmdb.Spec.Secrets.SSL = psmdbSplitHorizonSecret.GetName() + + return nil +} + +func getSplitHorizonDNSConfigSecretName(dbName string) string { + return dbName + "-sh-cert" +} + +// statusSplitHorizon returns true is split horizon DNS service is ready. +// It returns true if split horizon DNS is not configured. +func statusSplitHorizon(c *sdk.Context, psmdb *psmdbv1.PerconaServerMongoDB) error { + mongodSpec := &types.MongodCustomSpec{} + if err := c.DecodeComponentCustomSpec(c.DB().Spec.Components[ComponentEngine], mongodSpec); err != nil { + return fmt.Errorf("failed to decode mongod CustomSpec: %w", err) + } + + if mongodSpec.SplitHorizonDNSRef == nil { + // Nothing to do + return nil + } + + // Get generated external domains from PSMDB spec + for podName := range psmdb.Spec.Replsets[0].Horizons { + svc := &corev1.Service{} + if err := c.Get(svc, podName); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + return fmt.Errorf("failed to get service for SplitHorizon status: %w", err) + } + + // service not created yet + return fmt.Errorf("service %s not created yet", podName) + } + + // TODO only apply for service type of corev1.ServiceTypeExternalName + if _, ready := getServicePublicIP(svc); !ready { + return fmt.Errorf("service %s not ready", podName) + } + } + + return nil +} + +// getServicePublicIP retrieves the public IP address of the given service. +// It returns the public IP address and a boolean indicating whether the status is ready. +func getServicePublicIP(svc *corev1.Service) (string, bool) { + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + return "", false + } + + if len(svc.Status.LoadBalancer.Ingress) == 0 { + return publicIPPendingValue, false + } + + if svc.Status.LoadBalancer.Ingress[0].IP != "" { + return svc.Status.LoadBalancer.Ingress[0].IP, true + } + + // publicIP may be empty for load-balancer ingress points that are DNS based. + // In such case Hostname shall be set (typically AWS) + publicHostname := svc.Status.LoadBalancer.Ingress[0].Hostname + if publicHostname == "" { + return publicIPPendingValue, false + } + + // try to resolve DNS name to IP + var publicIPAddrs []net.IP + var err error + if publicIPAddrs, err = net.LookupIP(publicHostname); err != nil { + // Binding IP to domain takes some time, so just log a warning here. + fmt.Printf("resolve LoadBalancer ingress hostname %s to IP: %v\n", publicHostname, err) + return publicIPPendingValue, false + } + + for _, ip := range publicIPAddrs { + if ipStr := ip.String(); ipStr != "" { + return ipStr, true + } + } + + return publicIPPendingValue, false +} + +// ----------------- Helpers ----------------- +// issueSplitHorizonCertificate generates server TLS certificate signed by the provided CA certificate and private key, +// with SANs for the provided hosts. +// It returns the generated server TLS certificate, private key in PEM format and error. +func issueSplitHorizonCertificate(caCert, caPrivKey []byte, hosts []string) (string, string, error) { + caDecoded, _ := pem.Decode(caCert) + ca, err := x509.ParseCertificate(caDecoded.Bytes) + if err != nil { + return "", "", fmt.Errorf("parse CA certificate: %w", err) + } + + caPrivKeyDecoded, _ := pem.Decode(caPrivKey) + caKey, err := x509.ParsePKCS1PrivateKey(caPrivKeyDecoded.Bytes) + if err != nil { + return "", "", fmt.Errorf("parse CA private key: %w", err) + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) //nolint:mnd + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return "", "", fmt.Errorf("generate serial number for client: %w", err) + } + + serverCertTemplate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"PSMDB"}, + }, + NotBefore: time.Now(), + NotAfter: validityNotAfter, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, //nolint:mnd + DNSNames: hosts, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: false, + } + // Create server certificate private key + certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048) //nolint:mnd + if err != nil { + return "", "", fmt.Errorf("generate client key: %w", err) + } + + // Create and sign server certificate with CA + serverCertBytes, err := x509.CreateCertificate(rand.Reader, &serverCertTemplate, ca, &certPrivKey.PublicKey, caKey) + if err != nil { + return "", "", fmt.Errorf("generate server certificate: %w", err) + } + serverCertPem := &bytes.Buffer{} + err = pem.Encode(serverCertPem, &pem.Block{Type: "CERTIFICATE", Bytes: serverCertBytes}) + if err != nil { + return "", "", fmt.Errorf("encode server certificate: %w", err) + } + + serverKeyPem := &bytes.Buffer{} + block := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey)} + err = pem.Encode(serverKeyPem, block) + if err != nil { + return "", "", fmt.Errorf("encode RSA private key: %w", err) + } + + return serverCertPem.String(), serverKeyPem.String(), nil +} diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/00-assert.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/00-assert.yaml new file mode 100644 index 0000000..cacc348 --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/00-assert.yaml @@ -0,0 +1,58 @@ +apiVersion: kuttl.dev/v1 +kind: TestAssert +timeout: 10 +# collectors: +# - command: kubectl get splitdns/shdc-psmdb -n ${NAMESPACE} -o yaml +# - command: kubectl get secret/shdc-secret-exists -n ${NAMESPACE} -o yaml +# - command: kubectl get deploy/everest-controller-manager -n everest-system -o yaml +# - type: pod +# namespace: everest-system +# selector: control-plane=controller-manager +# tail: 100 +# resourceRefs: +# - apiVersion: enginefeatures.everest.percona.com/v1alpha1 +# kind: SplitHorizonDNSConfig +# name: shdc-psmdb +# ref: shdc +# assertAll: +# - celExpr: "!has(shdc.metadata.finalizers)" +# message: "shdc has unexpected finalizers" + +# - celExpr: "!has(shdc.spec.tls.certificate)" +# message: "shdc has unexpected .spec.tls.certificate" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: percona-server-mongodb-operator +spec: + replicas: 0 +# --- +# apiVersion: everest.percona.com/v1alpha1 +# kind: DatabaseEngine +# metadata: +# name: percona-server-mongodb-operator +# spec: +# type: psmdb +# status: +# status: installed +# --- +# apiVersion: enginefeatures.everest.percona.com/v1alpha1 +# kind: SplitHorizonDNSConfig +# metadata: +# name: shdc-psmdb +# spec: +# baseDomainNameSuffix: mycompany.com +# tls: +# secretName: shdc-secret-exists +# status: +# inUse: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: shdc-secret-exists +type: Opaque +data: + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmJZczVCa2xqSHRKckZkOFNnc0xhNzZvQ1FVQWdTcU1OdkZKVWxMSjBtMlM4eDI1ClZHWGFPWDM5Q1hHKzIxTzg5VGRoWHRRSU5nMEoxbTdmSlVFMDVGS2xKVTBPUWxRV2hETDNKWFF6UUdJc0FZV1YKV3Rib1VybjJvZVNoWHJmYkhOR3J4YnlmRitkcEJPRlpLY0pwU3Nta0ROMjVIVjlXbHdqdHFEVi9qZW92UHMvbgpPUm9VS0RJRjlSRUVwemtkbVNSbjlBNlBPb1A1UlJ6VkYzWTNlQ3JEK0RRbnQ3Qm5ndEtnWGZqbUg0RDJtSHZmCnBtWWJ0T3pjRzhNRXpBNGZaMldib2o3QlUxYTdZUmxiOVBGN1N1b1hBdldRMUJxNHlmWDN2em1KcjQ5RWZtRWoKbW4xZjdSeFdqV2M2QmxXaUFsVzFqeWJOc2VoZXl3MS9LSjk3V1FJREFRQUJBb0lCQUJZVzI1YUNGcEVMVEc1aQpuK1JUc2FSZHBsMXBmWS9zb3plbEErVnY0aFBNUWlWb2J6dEVZa0xhT0grMFpMV1BSQ1BWeTFMQ004UU9ZOWc1CjMrWHJ1Q1BET3pzekJBOWxVTnRhLzFPM09sMzdhRllUaHFyVmRlYm5CQkNBNldpNDFleWV2Y1pZQW5yRExVTmUKRlZhVFRsVUEwa2NVdkpDTzJLdGNwT1oyZHpnUDZpQ3BiUmNPUGVieDdHZXNiS2ZaRkxSOUNUSlgvUXY0YUFvMQpvL3BLYTZEeXk4V0MzNWtxOWQ3cTk4dmV1T3VMSWxPZ0k3UGdYMVBxUHJIOCs1OEI2YUduaC9Oa1VWMkd6OExKCkNSQmZoZW5RQ2EyVkM5UU9qLzhsbEVWdERTMXlNUEFUcG14ZWRESjI4ODNtWjRxNllaUjM3T0VVTVdUSWtzUWgKY0RNZUNlRUNnWUVBL0FSbldZUzFhYjQxeDVtcVlmbnhXMktvQmxvTm5LbWZGa1IvZEVqWE1ReEc3eFFSSTBNTAovenRHcUxxUHlaRGVNVGQ3ZzZVOHVmWWVPMURMQ3RjK3JwRlFVSWt1aHdxUEpVRVNiRUhMYzcvSkF5czREaUcvClVPWEJRd21RQmdlazVFaW9IaHhCNWQyWE4xTk5iemFnOFZsZkF4WDhlQnhFejFZa2RFZ0NQT0VDZ1lFQXdMV3oKRFFMaFNndW5pQjhZaXFmalQyaldZREF1NHY0ays5L1Izc1BkTm9xOVpCRk16VWE3N25pWFFOTUpFaVg5V2I2SQp2ZDBBdTAzdzdFYTNmbDIycWtmWHljQ1ZsYWxXVXJ5VjFyamxHS013VytEeEVHbUpYRWtEek1ycnRINU8xS0VCCmpFRWtPTC92Wkl6Z245UHNRSHNXTWdTbXFONDNmUCtjamdLYVZYa0NnWUJMRWZ5L083cldidVNTT29INGdYMlYKM1VOejhPdFJHVzNjTWpkTktrMS92TXA4ZVJ2Snh6VVJxRlNaK2tqT29DcXZ1bmsrYzhBdEhOVlFrZmFKbWloLwowNlY2K0FJMkU3MGtPY2dGRzJ4QlpJVzZQZXVLdWg3Rk9FdGpicnZLTUFpOFA3QmtsOEpCZU1xTW5uSFlpUXRVCkdXMGwvQ3lpa3Jra2tlSjJDT1V4d1FLQmdFUVVyUkh1cjRyS1BVQ1F3OG5RY0RUUXM5TzlrZ0x0aUVGWG5EeFgKOCtIZDkvVFBTOVBGcG9Va0kwTnFpdXpYY1A3d21qeUJSRTNueGpLaTlSWjJveDdiVExmaENyZVo0SDVRVTV0TgpMTnFjWkd4Qk5zajJqK21EZmcwdXIwRFAwcWU1emVNdjdFMEVPZDNMQzF4THNVNUNiZC96MXJFWCtJQjNpV1orCk11bjVBb0dCQUsyV3E0dFlOeWxQZVpiK2RiWWRUSmlPTkswYVdmZEs1WkpxSXdmcXNhb2xPdUIyVnhvVDVoWGkKQml4YWYwWFlPL3JlVUd1SjR5dXNQRzNOZEdEZ1h5cHFoWFU3Z2xRSmQ4SUMxMDh5elVPQjNOZy8xRGtkRjYrdwpPUWdLdmpvUXVaWDc0bkdUWG1lL3ZDSVhQRjB3Sy9IaFhhemZyWlIzMjc2VDkwQkUzdndECi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lVWTlLTWtlTC82RVhQaitWTjQ0N0hvWUY1TlpVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lqRU9NQXdHQTFVRUNoTUZVRk5OUkVJeEVEQU9CZ05WQkFNVEIxSnZiM1FnUTBFd0hoY05NalV4TVRBMQpNRGMxTmpBd1doY05NekF4TVRBME1EYzFOakF3V2pBaU1RNHdEQVlEVlFRS0V3VlFVMDFFUWpFUU1BNEdBMVVFCkF4TUhVbTl2ZENCRFFUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyMkxPUVoKSll4N1NheFhmRW9MQzJ1K3FBa0ZBSUVxakRieFNWSlN5ZEp0a3ZNZHVWUmwyamw5L1FseHZ0dFR2UFUzWVY3VQpDRFlOQ2RadTN5VkJOT1JTcFNWTkRrSlVGb1F5OXlWME0wQmlMQUdGbFZyVzZGSzU5cUhrb1Y2MzJ4elJxOFc4Cm54Zm5hUVRoV1NuQ2FVckpwQXpkdVIxZlZwY0k3YWcxZjQzcUx6N1A1emthRkNneUJmVVJCS2M1SFpra1ovUU8KanpxRCtVVWMxUmQyTjNncXcvZzBKN2V3WjRMU29GMzQ1aCtBOXBoNzM2Wm1HN1RzM0J2REJNd09IMmRsbTZJKwp3Vk5XdTJFWlcvVHhlMHJxRndMMWtOUWF1TW4xOTc4NWlhK1BSSDVoSTVwOVgrMGNWbzFuT2daVm9nSlZ0WThtCnpiSG9Yc3NOZnlpZmUxa0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdFR01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRzVXeWtBb2d0OG1hNzRTeXRxeXhxTS9TMm1NTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ2pmeU5oaDF2cjN3MVZXalRLajl4bUNrMU9YK1pBcVI3NkpkWkk0UHhnQ1hRQkNid1d6enBLCmtmeEFTTUowNE5LYnJFZ0tnWXczNG96clNvbVlMZndETm82czlLenBBZEQ2Z2F3Q0dPZXEzNzM4T3BSSU44YTkKNENZM0ZMRCs4emExcVI0ZkVtV1dOdnFvNlBNeDE3VGVjK092WnhJcTRXakZlRDRnZ3pYeWhqMTZzSGE2Z2YyYQphbjZvbktVVnhsYTdtTGdhY2taRzNXOTdOMlNBcXR3U252d3luYXNvaWFSeEM4OTRweC9FbnpNeXcwMElqeEFkCjZGMndWYVpqWmhBRDlRMkdIV29iS0RWK05LUDYxaXVWQjA4NVJZWWxVTXBxd3J4bkhBWllJQjk4SXJmVjFyRlUKQUhQOWRFNnVqMnFvTDQyWE5rN0ZvamJJRmw4RWIyRXQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/00-install.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/00-install.yaml new file mode 100644 index 0000000..dfe6b52 --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/00-install.yaml @@ -0,0 +1,34 @@ +apiVersion: kuttl.dev/v1 +kind: TestStep +timeout: 10 +commands: + # Need to patch KUTTL's namespace to add the label so that the Everest Operator can reconcile resources from it. + # - command: kubectl patch ns ${NAMESPACE} -p '{"metadata":{"labels":{"app.kubernetes.io/managed-by":"everest"}}}' --type merge + # We don't need the PSMDB operator to be running for the integration tests, but we need the deployment to exist otherwise the everest operator won't reconcile the CRs + - script: "curl -fsSL https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v${PSMDB_OPERATOR_VERSION}/deploy/operator.yaml | sed 's/replicas: [0-9]/replicas: 0/g' | kubectl apply -n ${NAMESPACE} -f -" +--- +apiVersion: v1 +kind: Secret +metadata: + name: shdc-secret-exists +type: Opaque +data: + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmJZczVCa2xqSHRKckZkOFNnc0xhNzZvQ1FVQWdTcU1OdkZKVWxMSjBtMlM4eDI1ClZHWGFPWDM5Q1hHKzIxTzg5VGRoWHRRSU5nMEoxbTdmSlVFMDVGS2xKVTBPUWxRV2hETDNKWFF6UUdJc0FZV1YKV3Rib1VybjJvZVNoWHJmYkhOR3J4YnlmRitkcEJPRlpLY0pwU3Nta0ROMjVIVjlXbHdqdHFEVi9qZW92UHMvbgpPUm9VS0RJRjlSRUVwemtkbVNSbjlBNlBPb1A1UlJ6VkYzWTNlQ3JEK0RRbnQ3Qm5ndEtnWGZqbUg0RDJtSHZmCnBtWWJ0T3pjRzhNRXpBNGZaMldib2o3QlUxYTdZUmxiOVBGN1N1b1hBdldRMUJxNHlmWDN2em1KcjQ5RWZtRWoKbW4xZjdSeFdqV2M2QmxXaUFsVzFqeWJOc2VoZXl3MS9LSjk3V1FJREFRQUJBb0lCQUJZVzI1YUNGcEVMVEc1aQpuK1JUc2FSZHBsMXBmWS9zb3plbEErVnY0aFBNUWlWb2J6dEVZa0xhT0grMFpMV1BSQ1BWeTFMQ004UU9ZOWc1CjMrWHJ1Q1BET3pzekJBOWxVTnRhLzFPM09sMzdhRllUaHFyVmRlYm5CQkNBNldpNDFleWV2Y1pZQW5yRExVTmUKRlZhVFRsVUEwa2NVdkpDTzJLdGNwT1oyZHpnUDZpQ3BiUmNPUGVieDdHZXNiS2ZaRkxSOUNUSlgvUXY0YUFvMQpvL3BLYTZEeXk4V0MzNWtxOWQ3cTk4dmV1T3VMSWxPZ0k3UGdYMVBxUHJIOCs1OEI2YUduaC9Oa1VWMkd6OExKCkNSQmZoZW5RQ2EyVkM5UU9qLzhsbEVWdERTMXlNUEFUcG14ZWRESjI4ODNtWjRxNllaUjM3T0VVTVdUSWtzUWgKY0RNZUNlRUNnWUVBL0FSbldZUzFhYjQxeDVtcVlmbnhXMktvQmxvTm5LbWZGa1IvZEVqWE1ReEc3eFFSSTBNTAovenRHcUxxUHlaRGVNVGQ3ZzZVOHVmWWVPMURMQ3RjK3JwRlFVSWt1aHdxUEpVRVNiRUhMYzcvSkF5czREaUcvClVPWEJRd21RQmdlazVFaW9IaHhCNWQyWE4xTk5iemFnOFZsZkF4WDhlQnhFejFZa2RFZ0NQT0VDZ1lFQXdMV3oKRFFMaFNndW5pQjhZaXFmalQyaldZREF1NHY0ays5L1Izc1BkTm9xOVpCRk16VWE3N25pWFFOTUpFaVg5V2I2SQp2ZDBBdTAzdzdFYTNmbDIycWtmWHljQ1ZsYWxXVXJ5VjFyamxHS013VytEeEVHbUpYRWtEek1ycnRINU8xS0VCCmpFRWtPTC92Wkl6Z245UHNRSHNXTWdTbXFONDNmUCtjamdLYVZYa0NnWUJMRWZ5L083cldidVNTT29INGdYMlYKM1VOejhPdFJHVzNjTWpkTktrMS92TXA4ZVJ2Snh6VVJxRlNaK2tqT29DcXZ1bmsrYzhBdEhOVlFrZmFKbWloLwowNlY2K0FJMkU3MGtPY2dGRzJ4QlpJVzZQZXVLdWg3Rk9FdGpicnZLTUFpOFA3QmtsOEpCZU1xTW5uSFlpUXRVCkdXMGwvQ3lpa3Jra2tlSjJDT1V4d1FLQmdFUVVyUkh1cjRyS1BVQ1F3OG5RY0RUUXM5TzlrZ0x0aUVGWG5EeFgKOCtIZDkvVFBTOVBGcG9Va0kwTnFpdXpYY1A3d21qeUJSRTNueGpLaTlSWjJveDdiVExmaENyZVo0SDVRVTV0TgpMTnFjWkd4Qk5zajJqK21EZmcwdXIwRFAwcWU1emVNdjdFMEVPZDNMQzF4THNVNUNiZC96MXJFWCtJQjNpV1orCk11bjVBb0dCQUsyV3E0dFlOeWxQZVpiK2RiWWRUSmlPTkswYVdmZEs1WkpxSXdmcXNhb2xPdUIyVnhvVDVoWGkKQml4YWYwWFlPL3JlVUd1SjR5dXNQRzNOZEdEZ1h5cHFoWFU3Z2xRSmQ4SUMxMDh5elVPQjNOZy8xRGtkRjYrdwpPUWdLdmpvUXVaWDc0bkdUWG1lL3ZDSVhQRjB3Sy9IaFhhemZyWlIzMjc2VDkwQkUzdndECi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lVWTlLTWtlTC82RVhQaitWTjQ0N0hvWUY1TlpVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lqRU9NQXdHQTFVRUNoTUZVRk5OUkVJeEVEQU9CZ05WQkFNVEIxSnZiM1FnUTBFd0hoY05NalV4TVRBMQpNRGMxTmpBd1doY05NekF4TVRBME1EYzFOakF3V2pBaU1RNHdEQVlEVlFRS0V3VlFVMDFFUWpFUU1BNEdBMVVFCkF4TUhVbTl2ZENCRFFUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyMkxPUVoKSll4N1NheFhmRW9MQzJ1K3FBa0ZBSUVxakRieFNWSlN5ZEp0a3ZNZHVWUmwyamw5L1FseHZ0dFR2UFUzWVY3VQpDRFlOQ2RadTN5VkJOT1JTcFNWTkRrSlVGb1F5OXlWME0wQmlMQUdGbFZyVzZGSzU5cUhrb1Y2MzJ4elJxOFc4Cm54Zm5hUVRoV1NuQ2FVckpwQXpkdVIxZlZwY0k3YWcxZjQzcUx6N1A1emthRkNneUJmVVJCS2M1SFpra1ovUU8KanpxRCtVVWMxUmQyTjNncXcvZzBKN2V3WjRMU29GMzQ1aCtBOXBoNzM2Wm1HN1RzM0J2REJNd09IMmRsbTZJKwp3Vk5XdTJFWlcvVHhlMHJxRndMMWtOUWF1TW4xOTc4NWlhK1BSSDVoSTVwOVgrMGNWbzFuT2daVm9nSlZ0WThtCnpiSG9Yc3NOZnlpZmUxa0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdFR01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRzVXeWtBb2d0OG1hNzRTeXRxeXhxTS9TMm1NTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ2pmeU5oaDF2cjN3MVZXalRLajl4bUNrMU9YK1pBcVI3NkpkWkk0UHhnQ1hRQkNid1d6enBLCmtmeEFTTUowNE5LYnJFZ0tnWXczNG96clNvbVlMZndETm82czlLenBBZEQ2Z2F3Q0dPZXEzNzM4T3BSSU44YTkKNENZM0ZMRCs4emExcVI0ZkVtV1dOdnFvNlBNeDE3VGVjK092WnhJcTRXakZlRDRnZ3pYeWhqMTZzSGE2Z2YyYQphbjZvbktVVnhsYTdtTGdhY2taRzNXOTdOMlNBcXR3U252d3luYXNvaWFSeEM4OTRweC9FbnpNeXcwMElqeEFkCjZGMndWYVpqWmhBRDlRMkdIV29iS0RWK05LUDYxaXVWQjA4NVJZWWxVTXBxd3J4bkhBWllJQjk4SXJmVjFyRlUKQUhQOWRFNnVqMnFvTDQyWE5rN0ZvamJJRmw4RWIyRXQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +# --- +# apiVersion: enginefeatures.everest.percona.com/v1alpha1 +# kind: SplitHorizonDNSConfig +# metadata: +# name: shdc-psmdb +# spec: +# baseDomainNameSuffix: mycompany.com +# tls: +# secretName: shdc-secret-exists +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: shdc-psmdb +data: + baseDomainNameSuffix: mycompany.com + secretName: shdc-secret-exists \ No newline at end of file diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/10-assert.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/10-assert.yaml new file mode 100644 index 0000000..277fe82 --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/10-assert.yaml @@ -0,0 +1,104 @@ +apiVersion: kuttl.dev/v1 +kind: TestAssert +timeout: 30 +collectors: + # - command: kubectl get splitdns/shdc-psmdb -n ${NAMESPACE} -o yaml + - command: kubectl get secret/shdc-secret-exists -n ${NAMESPACE} -o yaml + # - command: kubectl get db/psmdb-shdc -n ${NAMESPACE} -o yaml + - command: kubectl get psmdb/psmdb-shdc -n ${NAMESPACE} -o yaml + # - type: pod + # namespace: everest-system + # selector: control-plane=controller-manager + # tail: 100 +resourceRefs: + # - apiVersion: enginefeatures.everest.percona.com/v1alpha1 + # kind: SplitHorizonDNSConfig + # name: shdc-psmdb + # ref: shdc + # - apiVersion: everest.percona.com/v1alpha1 + # kind: DatabaseCluster + # name: psmdb-shdc + # ref: db + - apiVersion: psmdb.percona.com/v1 + kind: PerconaServerMongoDB + name: psmdb-shdc + ref: psmdb + - apiVersion: everest.percona.com/v2alpha1 + kind: DataStore + name: psmdb-shdc + ref: dst +assertAll: + # - celExpr: "has(shdc.metadata.finalizers)" + # message: "shdc doesn't have finalizers" + + # - celExpr: "!shdc.metadata.finalizers.exists(k, k == 'everest.percona.com/cleanup-secrets')" + # message: "'everest.percona.com/cleanup-secrets' presents in shdc.metadata.finalizers" + + # - celExpr: "'everest.percona.com/in-use-protection' in shdc.metadata.finalizers" + # message: "'everest.percona.com/in-use-protection' is absent in shdc.metadata.finalizers" + + # - celExpr: "!has(shdc.spec.tls.certificate)" + # message: "shdc has unexpected .spec.tls.certificate" + + - celExpr: "has(psmdb.spec.replsets[0].splitHorizons)" + message: "psmdb.spec.replsets[0].splitHorizons is empty" + + - celExpr: "psmdb.spec.replsets[0].splitHorizons.exists(k, k == 'psmdb-shdc-rs0-0')" + message: "psmdb.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-0 is empty" + + - celExpr: "psmdb.spec.replsets[0].splitHorizons['psmdb-shdc-rs0-0'].exists(k, k == 'external')" + message: "psmdb.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-0.external is empty" + + - celExpr: "psmdb.spec.replsets[0].splitHorizons.exists(k, k == 'psmdb-shdc-rs0-1')" + message: "psmdb.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-1 is empty" + + - celExpr: "psmdb.spec.replsets[0].splitHorizons['psmdb-shdc-rs0-1'].exists(k, k == 'external')" + message: "psmdb.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-1.external is empty" + + - celExpr: "psmdb.spec.replsets[0].splitHorizons.exists(k, k == 'psmdb-shdc-rs0-2')" + message: "psmdb.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-2 is empty" + + - celExpr: "psmdb.spec.replsets[0].splitHorizons['psmdb-shdc-rs0-2'].exists(k, k == 'external')" + message: "psmdb.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-2.external is empty" +commands: + - command: kubectl wait --for=jsonpath='{.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-0.external}'=psmdb-shdc-rs0-0-${NAMESPACE}.mycompany.com psmdb/psmdb-shdc -n ${NAMESPACE} + - command: kubectl wait --for=jsonpath='{.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-1.external}'=psmdb-shdc-rs0-1-${NAMESPACE}.mycompany.com psmdb/psmdb-shdc -n ${NAMESPACE} + - command: kubectl wait --for=jsonpath='{.spec.replsets[0].splitHorizons.psmdb-shdc-rs0-2.external}'=psmdb-shdc-rs0-2-${NAMESPACE}.mycompany.com psmdb/psmdb-shdc -n ${NAMESPACE} +# --- +# apiVersion: enginefeatures.everest.percona.com/v1alpha1 +# kind: SplitHorizonDNSConfig +# metadata: +# name: shdc-psmdb +# spec: +# baseDomainNameSuffix: mycompany.com +# tls: +# secretName: shdc-secret-exists +# status: +# inUse: true +--- +apiVersion: v1 +kind: Secret +metadata: + name: shdc-secret-exists +type: Opaque +data: + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmJZczVCa2xqSHRKckZkOFNnc0xhNzZvQ1FVQWdTcU1OdkZKVWxMSjBtMlM4eDI1ClZHWGFPWDM5Q1hHKzIxTzg5VGRoWHRRSU5nMEoxbTdmSlVFMDVGS2xKVTBPUWxRV2hETDNKWFF6UUdJc0FZV1YKV3Rib1VybjJvZVNoWHJmYkhOR3J4YnlmRitkcEJPRlpLY0pwU3Nta0ROMjVIVjlXbHdqdHFEVi9qZW92UHMvbgpPUm9VS0RJRjlSRUVwemtkbVNSbjlBNlBPb1A1UlJ6VkYzWTNlQ3JEK0RRbnQ3Qm5ndEtnWGZqbUg0RDJtSHZmCnBtWWJ0T3pjRzhNRXpBNGZaMldib2o3QlUxYTdZUmxiOVBGN1N1b1hBdldRMUJxNHlmWDN2em1KcjQ5RWZtRWoKbW4xZjdSeFdqV2M2QmxXaUFsVzFqeWJOc2VoZXl3MS9LSjk3V1FJREFRQUJBb0lCQUJZVzI1YUNGcEVMVEc1aQpuK1JUc2FSZHBsMXBmWS9zb3plbEErVnY0aFBNUWlWb2J6dEVZa0xhT0grMFpMV1BSQ1BWeTFMQ004UU9ZOWc1CjMrWHJ1Q1BET3pzekJBOWxVTnRhLzFPM09sMzdhRllUaHFyVmRlYm5CQkNBNldpNDFleWV2Y1pZQW5yRExVTmUKRlZhVFRsVUEwa2NVdkpDTzJLdGNwT1oyZHpnUDZpQ3BiUmNPUGVieDdHZXNiS2ZaRkxSOUNUSlgvUXY0YUFvMQpvL3BLYTZEeXk4V0MzNWtxOWQ3cTk4dmV1T3VMSWxPZ0k3UGdYMVBxUHJIOCs1OEI2YUduaC9Oa1VWMkd6OExKCkNSQmZoZW5RQ2EyVkM5UU9qLzhsbEVWdERTMXlNUEFUcG14ZWRESjI4ODNtWjRxNllaUjM3T0VVTVdUSWtzUWgKY0RNZUNlRUNnWUVBL0FSbldZUzFhYjQxeDVtcVlmbnhXMktvQmxvTm5LbWZGa1IvZEVqWE1ReEc3eFFSSTBNTAovenRHcUxxUHlaRGVNVGQ3ZzZVOHVmWWVPMURMQ3RjK3JwRlFVSWt1aHdxUEpVRVNiRUhMYzcvSkF5czREaUcvClVPWEJRd21RQmdlazVFaW9IaHhCNWQyWE4xTk5iemFnOFZsZkF4WDhlQnhFejFZa2RFZ0NQT0VDZ1lFQXdMV3oKRFFMaFNndW5pQjhZaXFmalQyaldZREF1NHY0ays5L1Izc1BkTm9xOVpCRk16VWE3N25pWFFOTUpFaVg5V2I2SQp2ZDBBdTAzdzdFYTNmbDIycWtmWHljQ1ZsYWxXVXJ5VjFyamxHS013VytEeEVHbUpYRWtEek1ycnRINU8xS0VCCmpFRWtPTC92Wkl6Z245UHNRSHNXTWdTbXFONDNmUCtjamdLYVZYa0NnWUJMRWZ5L083cldidVNTT29INGdYMlYKM1VOejhPdFJHVzNjTWpkTktrMS92TXA4ZVJ2Snh6VVJxRlNaK2tqT29DcXZ1bmsrYzhBdEhOVlFrZmFKbWloLwowNlY2K0FJMkU3MGtPY2dGRzJ4QlpJVzZQZXVLdWg3Rk9FdGpicnZLTUFpOFA3QmtsOEpCZU1xTW5uSFlpUXRVCkdXMGwvQ3lpa3Jra2tlSjJDT1V4d1FLQmdFUVVyUkh1cjRyS1BVQ1F3OG5RY0RUUXM5TzlrZ0x0aUVGWG5EeFgKOCtIZDkvVFBTOVBGcG9Va0kwTnFpdXpYY1A3d21qeUJSRTNueGpLaTlSWjJveDdiVExmaENyZVo0SDVRVTV0TgpMTnFjWkd4Qk5zajJqK21EZmcwdXIwRFAwcWU1emVNdjdFMEVPZDNMQzF4THNVNUNiZC96MXJFWCtJQjNpV1orCk11bjVBb0dCQUsyV3E0dFlOeWxQZVpiK2RiWWRUSmlPTkswYVdmZEs1WkpxSXdmcXNhb2xPdUIyVnhvVDVoWGkKQml4YWYwWFlPL3JlVUd1SjR5dXNQRzNOZEdEZ1h5cHFoWFU3Z2xRSmQ4SUMxMDh5elVPQjNOZy8xRGtkRjYrdwpPUWdLdmpvUXVaWDc0bkdUWG1lL3ZDSVhQRjB3Sy9IaFhhemZyWlIzMjc2VDkwQkUzdndECi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lVWTlLTWtlTC82RVhQaitWTjQ0N0hvWUY1TlpVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lqRU9NQXdHQTFVRUNoTUZVRk5OUkVJeEVEQU9CZ05WQkFNVEIxSnZiM1FnUTBFd0hoY05NalV4TVRBMQpNRGMxTmpBd1doY05NekF4TVRBME1EYzFOakF3V2pBaU1RNHdEQVlEVlFRS0V3VlFVMDFFUWpFUU1BNEdBMVVFCkF4TUhVbTl2ZENCRFFUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyMkxPUVoKSll4N1NheFhmRW9MQzJ1K3FBa0ZBSUVxakRieFNWSlN5ZEp0a3ZNZHVWUmwyamw5L1FseHZ0dFR2UFUzWVY3VQpDRFlOQ2RadTN5VkJOT1JTcFNWTkRrSlVGb1F5OXlWME0wQmlMQUdGbFZyVzZGSzU5cUhrb1Y2MzJ4elJxOFc4Cm54Zm5hUVRoV1NuQ2FVckpwQXpkdVIxZlZwY0k3YWcxZjQzcUx6N1A1emthRkNneUJmVVJCS2M1SFpra1ovUU8KanpxRCtVVWMxUmQyTjNncXcvZzBKN2V3WjRMU29GMzQ1aCtBOXBoNzM2Wm1HN1RzM0J2REJNd09IMmRsbTZJKwp3Vk5XdTJFWlcvVHhlMHJxRndMMWtOUWF1TW4xOTc4NWlhK1BSSDVoSTVwOVgrMGNWbzFuT2daVm9nSlZ0WThtCnpiSG9Yc3NOZnlpZmUxa0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdFR01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRzVXeWtBb2d0OG1hNzRTeXRxeXhxTS9TMm1NTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ2pmeU5oaDF2cjN3MVZXalRLajl4bUNrMU9YK1pBcVI3NkpkWkk0UHhnQ1hRQkNid1d6enBLCmtmeEFTTUowNE5LYnJFZ0tnWXczNG96clNvbVlMZndETm82czlLenBBZEQ2Z2F3Q0dPZXEzNzM4T3BSSU44YTkKNENZM0ZMRCs4emExcVI0ZkVtV1dOdnFvNlBNeDE3VGVjK092WnhJcTRXakZlRDRnZ3pYeWhqMTZzSGE2Z2YyYQphbjZvbktVVnhsYTdtTGdhY2taRzNXOTdOMlNBcXR3U252d3luYXNvaWFSeEM4OTRweC9FbnpNeXcwMElqeEFkCjZGMndWYVpqWmhBRDlRMkdIV29iS0RWK05LUDYxaXVWQjA4NVJZWWxVTXBxd3J4bkhBWllJQjk4SXJmVjFyRlUKQUhQOWRFNnVqMnFvTDQyWE5rN0ZvamJJRmw4RWIyRXQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +# --- +# apiVersion: everest.percona.com/v1alpha1 +# kind: DatabaseCluster +# metadata: +# name: psmdb-shdc +# spec: +# engineFeatures: +# psmdb: +# splitHorizonDnsConfigName: shdc-psmdb +--- +apiVersion: psmdb.percona.com/v1 +kind: PerconaServerMongoDB +metadata: + name: psmdb-shdc +spec: + secrets: + ssl: psmdb-shdc-sh-cert + sslInternal: psmdb-shdc-ssl-internal diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/10-create-cluster.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/10-create-cluster.yaml new file mode 100644 index 0000000..6f38344 --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/10-create-cluster.yaml @@ -0,0 +1,49 @@ +apiVersion: kuttl.dev/v1 +kind: TestStep +--- +apiVersion: everest.percona.com/v2alpha1 +kind: DataStore +metadata: + name: psmdb-shdc +spec: + provider: psmdb + components: + engine: + type: mongod + replicas: 3 + resources: + cpu: "1" + memory: 4G + storage: + size: 1Gi + customSpec: + splitHorizonDNSRef: + name: shdc-psmdb + proxy: + type: mongos + version: 8.0.8-3 + replicas: 1 +# kind: DatabaseCluster +# metadata: +# name: psmdb-shdc +# spec: +# engine: +# replicas: 3 +# resources: +# cpu: "1" +# memory: 4G +# storage: +# class: local-path +# size: 1Gi +# type: psmdb +# engineFeatures: +# psmdb: +# splitHorizonDnsConfigName: shdc-psmdb +# proxy: +# expose: +# type: internal +# replicas: 1 +# resources: +# cpu: "1" +# memory: 2G +# type: mongos diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/20-assert.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/20-assert.yaml new file mode 100644 index 0000000..fee7753 --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/20-assert.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1 +kind: TestAssert +timeout: 10 +collectors: + # - command: kubectl get splitdns/shdc-psmdb -n ${NAMESPACE} -o yaml + - command: kubectl get secret/shdc-secret-exists -n ${NAMESPACE} -o yaml + # - command: kubectl get db/psmdb-shdc -n ${NAMESPACE} -o yaml + - command: kubectl get psmdb/psmdb-shdc -n ${NAMESPACE} -o yaml + # - type: pod + # namespace: everest-system + # selector: control-plane=controller-manager + # tail: 100 +commands: + # - command: kubectl wait --for=delete db/psmdb-shdc -n ${NAMESPACE} + - command: kubectl wait --for=delete dst/psmdb-shdc -n ${NAMESPACE} +resourceRefs: + # - apiVersion: enginefeatures.everest.percona.com/v1alpha1 + # kind: SplitHorizonDNSConfig + # name: shdc-psmdb + # ref: shdc +assertAll: + # - celExpr: "!has(shdc.metadata.finalizers)" + # message: "shdc doesn't have finalizers" + + # - celExpr: "!has(shdc.spec.tls.certificate)" + # message: "shdc has unexpected .spec.tls.certificate" +# --- +# apiVersion: enginefeatures.everest.percona.com/v1alpha1 +# kind: SplitHorizonDNSConfig +# metadata: +# name: shdc-psmdb +# spec: +# baseDomainNameSuffix: mycompany.com +# tls: +# secretName: shdc-secret-exists +# status: +# inUse: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: shdc-secret-exists +type: Opaque +data: + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmJZczVCa2xqSHRKckZkOFNnc0xhNzZvQ1FVQWdTcU1OdkZKVWxMSjBtMlM4eDI1ClZHWGFPWDM5Q1hHKzIxTzg5VGRoWHRRSU5nMEoxbTdmSlVFMDVGS2xKVTBPUWxRV2hETDNKWFF6UUdJc0FZV1YKV3Rib1VybjJvZVNoWHJmYkhOR3J4YnlmRitkcEJPRlpLY0pwU3Nta0ROMjVIVjlXbHdqdHFEVi9qZW92UHMvbgpPUm9VS0RJRjlSRUVwemtkbVNSbjlBNlBPb1A1UlJ6VkYzWTNlQ3JEK0RRbnQ3Qm5ndEtnWGZqbUg0RDJtSHZmCnBtWWJ0T3pjRzhNRXpBNGZaMldib2o3QlUxYTdZUmxiOVBGN1N1b1hBdldRMUJxNHlmWDN2em1KcjQ5RWZtRWoKbW4xZjdSeFdqV2M2QmxXaUFsVzFqeWJOc2VoZXl3MS9LSjk3V1FJREFRQUJBb0lCQUJZVzI1YUNGcEVMVEc1aQpuK1JUc2FSZHBsMXBmWS9zb3plbEErVnY0aFBNUWlWb2J6dEVZa0xhT0grMFpMV1BSQ1BWeTFMQ004UU9ZOWc1CjMrWHJ1Q1BET3pzekJBOWxVTnRhLzFPM09sMzdhRllUaHFyVmRlYm5CQkNBNldpNDFleWV2Y1pZQW5yRExVTmUKRlZhVFRsVUEwa2NVdkpDTzJLdGNwT1oyZHpnUDZpQ3BiUmNPUGVieDdHZXNiS2ZaRkxSOUNUSlgvUXY0YUFvMQpvL3BLYTZEeXk4V0MzNWtxOWQ3cTk4dmV1T3VMSWxPZ0k3UGdYMVBxUHJIOCs1OEI2YUduaC9Oa1VWMkd6OExKCkNSQmZoZW5RQ2EyVkM5UU9qLzhsbEVWdERTMXlNUEFUcG14ZWRESjI4ODNtWjRxNllaUjM3T0VVTVdUSWtzUWgKY0RNZUNlRUNnWUVBL0FSbldZUzFhYjQxeDVtcVlmbnhXMktvQmxvTm5LbWZGa1IvZEVqWE1ReEc3eFFSSTBNTAovenRHcUxxUHlaRGVNVGQ3ZzZVOHVmWWVPMURMQ3RjK3JwRlFVSWt1aHdxUEpVRVNiRUhMYzcvSkF5czREaUcvClVPWEJRd21RQmdlazVFaW9IaHhCNWQyWE4xTk5iemFnOFZsZkF4WDhlQnhFejFZa2RFZ0NQT0VDZ1lFQXdMV3oKRFFMaFNndW5pQjhZaXFmalQyaldZREF1NHY0ays5L1Izc1BkTm9xOVpCRk16VWE3N25pWFFOTUpFaVg5V2I2SQp2ZDBBdTAzdzdFYTNmbDIycWtmWHljQ1ZsYWxXVXJ5VjFyamxHS013VytEeEVHbUpYRWtEek1ycnRINU8xS0VCCmpFRWtPTC92Wkl6Z245UHNRSHNXTWdTbXFONDNmUCtjamdLYVZYa0NnWUJMRWZ5L083cldidVNTT29INGdYMlYKM1VOejhPdFJHVzNjTWpkTktrMS92TXA4ZVJ2Snh6VVJxRlNaK2tqT29DcXZ1bmsrYzhBdEhOVlFrZmFKbWloLwowNlY2K0FJMkU3MGtPY2dGRzJ4QlpJVzZQZXVLdWg3Rk9FdGpicnZLTUFpOFA3QmtsOEpCZU1xTW5uSFlpUXRVCkdXMGwvQ3lpa3Jra2tlSjJDT1V4d1FLQmdFUVVyUkh1cjRyS1BVQ1F3OG5RY0RUUXM5TzlrZ0x0aUVGWG5EeFgKOCtIZDkvVFBTOVBGcG9Va0kwTnFpdXpYY1A3d21qeUJSRTNueGpLaTlSWjJveDdiVExmaENyZVo0SDVRVTV0TgpMTnFjWkd4Qk5zajJqK21EZmcwdXIwRFAwcWU1emVNdjdFMEVPZDNMQzF4THNVNUNiZC96MXJFWCtJQjNpV1orCk11bjVBb0dCQUsyV3E0dFlOeWxQZVpiK2RiWWRUSmlPTkswYVdmZEs1WkpxSXdmcXNhb2xPdUIyVnhvVDVoWGkKQml4YWYwWFlPL3JlVUd1SjR5dXNQRzNOZEdEZ1h5cHFoWFU3Z2xRSmQ4SUMxMDh5elVPQjNOZy8xRGtkRjYrdwpPUWdLdmpvUXVaWDc0bkdUWG1lL3ZDSVhQRjB3Sy9IaFhhemZyWlIzMjc2VDkwQkUzdndECi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lVWTlLTWtlTC82RVhQaitWTjQ0N0hvWUY1TlpVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lqRU9NQXdHQTFVRUNoTUZVRk5OUkVJeEVEQU9CZ05WQkFNVEIxSnZiM1FnUTBFd0hoY05NalV4TVRBMQpNRGMxTmpBd1doY05NekF4TVRBME1EYzFOakF3V2pBaU1RNHdEQVlEVlFRS0V3VlFVMDFFUWpFUU1BNEdBMVVFCkF4TUhVbTl2ZENCRFFUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyMkxPUVoKSll4N1NheFhmRW9MQzJ1K3FBa0ZBSUVxakRieFNWSlN5ZEp0a3ZNZHVWUmwyamw5L1FseHZ0dFR2UFUzWVY3VQpDRFlOQ2RadTN5VkJOT1JTcFNWTkRrSlVGb1F5OXlWME0wQmlMQUdGbFZyVzZGSzU5cUhrb1Y2MzJ4elJxOFc4Cm54Zm5hUVRoV1NuQ2FVckpwQXpkdVIxZlZwY0k3YWcxZjQzcUx6N1A1emthRkNneUJmVVJCS2M1SFpra1ovUU8KanpxRCtVVWMxUmQyTjNncXcvZzBKN2V3WjRMU29GMzQ1aCtBOXBoNzM2Wm1HN1RzM0J2REJNd09IMmRsbTZJKwp3Vk5XdTJFWlcvVHhlMHJxRndMMWtOUWF1TW4xOTc4NWlhK1BSSDVoSTVwOVgrMGNWbzFuT2daVm9nSlZ0WThtCnpiSG9Yc3NOZnlpZmUxa0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdFR01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRzVXeWtBb2d0OG1hNzRTeXRxeXhxTS9TMm1NTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ2pmeU5oaDF2cjN3MVZXalRLajl4bUNrMU9YK1pBcVI3NkpkWkk0UHhnQ1hRQkNid1d6enBLCmtmeEFTTUowNE5LYnJFZ0tnWXczNG96clNvbVlMZndETm82czlLenBBZEQ2Z2F3Q0dPZXEzNzM4T3BSSU44YTkKNENZM0ZMRCs4emExcVI0ZkVtV1dOdnFvNlBNeDE3VGVjK092WnhJcTRXakZlRDRnZ3pYeWhqMTZzSGE2Z2YyYQphbjZvbktVVnhsYTdtTGdhY2taRzNXOTdOMlNBcXR3U252d3luYXNvaWFSeEM4OTRweC9FbnpNeXcwMElqeEFkCjZGMndWYVpqWmhBRDlRMkdIV29iS0RWK05LUDYxaXVWQjA4NVJZWWxVTXBxd3J4bkhBWllJQjk4SXJmVjFyRlUKQUhQOWRFNnVqMnFvTDQyWE5rN0ZvamJJRmw4RWIyRXQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/20-delete-cluster.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/20-delete-cluster.yaml new file mode 100644 index 0000000..46d7aec --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/20-delete-cluster.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1 +kind: TestStep +timeout: 30 +commands: + #- script: kubectl -n $NAMESPACE delete db/psmdb-shdc psmdb/psmdb-shdc --wait=false && sleep 5 + - script: kubectl -n $NAMESPACE delete dst/psmdb-shdc psmdb/psmdb-shdc --wait=false && sleep 5 + # - command: kubectl patch db/psmdb-shdc -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge + # - command: kubectl patch psmdb/psmdb-shdc -n $NAMESPACE -p '{"metadata":{"finalizers":null}}' --type merge diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/30-assert.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/30-assert.yaml new file mode 100644 index 0000000..f67b0ad --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/30-assert.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1 +kind: TestAssert +timeout: 10 +collectors: + # - command: kubectl get splitdns/shdc-psmdb -n ${NAMESPACE} -o yaml + - command: kubectl get secret/shdc-secret-exists -n ${NAMESPACE} -o yaml + # - type: pod + # namespace: everest-system + # selector: control-plane=controller-manager + # tail: 100 +commands: + # - command: kubectl wait --for=delete splitdns/shdc-psmdb -n ${NAMESPACE} +--- +apiVersion: v1 +kind: Secret +metadata: + name: shdc-secret-exists +type: Opaque +data: + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmJZczVCa2xqSHRKckZkOFNnc0xhNzZvQ1FVQWdTcU1OdkZKVWxMSjBtMlM4eDI1ClZHWGFPWDM5Q1hHKzIxTzg5VGRoWHRRSU5nMEoxbTdmSlVFMDVGS2xKVTBPUWxRV2hETDNKWFF6UUdJc0FZV1YKV3Rib1VybjJvZVNoWHJmYkhOR3J4YnlmRitkcEJPRlpLY0pwU3Nta0ROMjVIVjlXbHdqdHFEVi9qZW92UHMvbgpPUm9VS0RJRjlSRUVwemtkbVNSbjlBNlBPb1A1UlJ6VkYzWTNlQ3JEK0RRbnQ3Qm5ndEtnWGZqbUg0RDJtSHZmCnBtWWJ0T3pjRzhNRXpBNGZaMldib2o3QlUxYTdZUmxiOVBGN1N1b1hBdldRMUJxNHlmWDN2em1KcjQ5RWZtRWoKbW4xZjdSeFdqV2M2QmxXaUFsVzFqeWJOc2VoZXl3MS9LSjk3V1FJREFRQUJBb0lCQUJZVzI1YUNGcEVMVEc1aQpuK1JUc2FSZHBsMXBmWS9zb3plbEErVnY0aFBNUWlWb2J6dEVZa0xhT0grMFpMV1BSQ1BWeTFMQ004UU9ZOWc1CjMrWHJ1Q1BET3pzekJBOWxVTnRhLzFPM09sMzdhRllUaHFyVmRlYm5CQkNBNldpNDFleWV2Y1pZQW5yRExVTmUKRlZhVFRsVUEwa2NVdkpDTzJLdGNwT1oyZHpnUDZpQ3BiUmNPUGVieDdHZXNiS2ZaRkxSOUNUSlgvUXY0YUFvMQpvL3BLYTZEeXk4V0MzNWtxOWQ3cTk4dmV1T3VMSWxPZ0k3UGdYMVBxUHJIOCs1OEI2YUduaC9Oa1VWMkd6OExKCkNSQmZoZW5RQ2EyVkM5UU9qLzhsbEVWdERTMXlNUEFUcG14ZWRESjI4ODNtWjRxNllaUjM3T0VVTVdUSWtzUWgKY0RNZUNlRUNnWUVBL0FSbldZUzFhYjQxeDVtcVlmbnhXMktvQmxvTm5LbWZGa1IvZEVqWE1ReEc3eFFSSTBNTAovenRHcUxxUHlaRGVNVGQ3ZzZVOHVmWWVPMURMQ3RjK3JwRlFVSWt1aHdxUEpVRVNiRUhMYzcvSkF5czREaUcvClVPWEJRd21RQmdlazVFaW9IaHhCNWQyWE4xTk5iemFnOFZsZkF4WDhlQnhFejFZa2RFZ0NQT0VDZ1lFQXdMV3oKRFFMaFNndW5pQjhZaXFmalQyaldZREF1NHY0ays5L1Izc1BkTm9xOVpCRk16VWE3N25pWFFOTUpFaVg5V2I2SQp2ZDBBdTAzdzdFYTNmbDIycWtmWHljQ1ZsYWxXVXJ5VjFyamxHS013VytEeEVHbUpYRWtEek1ycnRINU8xS0VCCmpFRWtPTC92Wkl6Z245UHNRSHNXTWdTbXFONDNmUCtjamdLYVZYa0NnWUJMRWZ5L083cldidVNTT29INGdYMlYKM1VOejhPdFJHVzNjTWpkTktrMS92TXA4ZVJ2Snh6VVJxRlNaK2tqT29DcXZ1bmsrYzhBdEhOVlFrZmFKbWloLwowNlY2K0FJMkU3MGtPY2dGRzJ4QlpJVzZQZXVLdWg3Rk9FdGpicnZLTUFpOFA3QmtsOEpCZU1xTW5uSFlpUXRVCkdXMGwvQ3lpa3Jra2tlSjJDT1V4d1FLQmdFUVVyUkh1cjRyS1BVQ1F3OG5RY0RUUXM5TzlrZ0x0aUVGWG5EeFgKOCtIZDkvVFBTOVBGcG9Va0kwTnFpdXpYY1A3d21qeUJSRTNueGpLaTlSWjJveDdiVExmaENyZVo0SDVRVTV0TgpMTnFjWkd4Qk5zajJqK21EZmcwdXIwRFAwcWU1emVNdjdFMEVPZDNMQzF4THNVNUNiZC96MXJFWCtJQjNpV1orCk11bjVBb0dCQUsyV3E0dFlOeWxQZVpiK2RiWWRUSmlPTkswYVdmZEs1WkpxSXdmcXNhb2xPdUIyVnhvVDVoWGkKQml4YWYwWFlPL3JlVUd1SjR5dXNQRzNOZEdEZ1h5cHFoWFU3Z2xRSmQ4SUMxMDh5elVPQjNOZy8xRGtkRjYrdwpPUWdLdmpvUXVaWDc0bkdUWG1lL3ZDSVhQRjB3Sy9IaFhhemZyWlIzMjc2VDkwQkUzdndECi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lVWTlLTWtlTC82RVhQaitWTjQ0N0hvWUY1TlpVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0lqRU9NQXdHQTFVRUNoTUZVRk5OUkVJeEVEQU9CZ05WQkFNVEIxSnZiM1FnUTBFd0hoY05NalV4TVRBMQpNRGMxTmpBd1doY05NekF4TVRBME1EYzFOakF3V2pBaU1RNHdEQVlEVlFRS0V3VlFVMDFFUWpFUU1BNEdBMVVFCkF4TUhVbTl2ZENCRFFUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyMkxPUVoKSll4N1NheFhmRW9MQzJ1K3FBa0ZBSUVxakRieFNWSlN5ZEp0a3ZNZHVWUmwyamw5L1FseHZ0dFR2UFUzWVY3VQpDRFlOQ2RadTN5VkJOT1JTcFNWTkRrSlVGb1F5OXlWME0wQmlMQUdGbFZyVzZGSzU5cUhrb1Y2MzJ4elJxOFc4Cm54Zm5hUVRoV1NuQ2FVckpwQXpkdVIxZlZwY0k3YWcxZjQzcUx6N1A1emthRkNneUJmVVJCS2M1SFpra1ovUU8KanpxRCtVVWMxUmQyTjNncXcvZzBKN2V3WjRMU29GMzQ1aCtBOXBoNzM2Wm1HN1RzM0J2REJNd09IMmRsbTZJKwp3Vk5XdTJFWlcvVHhlMHJxRndMMWtOUWF1TW4xOTc4NWlhK1BSSDVoSTVwOVgrMGNWbzFuT2daVm9nSlZ0WThtCnpiSG9Yc3NOZnlpZmUxa0NBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdFR01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRzVXeWtBb2d0OG1hNzRTeXRxeXhxTS9TMm1NTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ2pmeU5oaDF2cjN3MVZXalRLajl4bUNrMU9YK1pBcVI3NkpkWkk0UHhnQ1hRQkNid1d6enBLCmtmeEFTTUowNE5LYnJFZ0tnWXczNG96clNvbVlMZndETm82czlLenBBZEQ2Z2F3Q0dPZXEzNzM4T3BSSU44YTkKNENZM0ZMRCs4emExcVI0ZkVtV1dOdnFvNlBNeDE3VGVjK092WnhJcTRXakZlRDRnZ3pYeWhqMTZzSGE2Z2YyYQphbjZvbktVVnhsYTdtTGdhY2taRzNXOTdOMlNBcXR3U252d3luYXNvaWFSeEM4OTRweC9FbnpNeXcwMElqeEFkCjZGMndWYVpqWmhBRDlRMkdIV29iS0RWK05LUDYxaXVWQjA4NVJZWWxVTXBxd3J4bkhBWllJQjk4SXJmVjFyRlUKQUhQOWRFNnVqMnFvTDQyWE5rN0ZvamJJRmw4RWIyRXQKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/examples/psmdb/test/integration/split_horizon_existing_secret/30-cleanup.yaml b/examples/psmdb/test/integration/split_horizon_existing_secret/30-cleanup.yaml new file mode 100644 index 0000000..9f7db06 --- /dev/null +++ b/examples/psmdb/test/integration/split_horizon_existing_secret/30-cleanup.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1 +kind: TestStep +timeout: 10 +delete: + # - apiVersion: enginefeatures.everest.percona.com/v1alpha1 + # kind: SplitHorizonDNSConfig + # name: shdc-psmdb + - apiVersion: v1 + kind: ConfigMap + name: shdc-psmdb \ No newline at end of file diff --git a/examples/psmdb/types/types.go b/examples/psmdb/types/types.go index 66f738e..4d0db23 100644 --- a/examples/psmdb/types/types.go +++ b/examples/psmdb/types/types.go @@ -33,14 +33,24 @@ package types // MongodCustomSpec defines custom configuration for mongod components. // This struct is converted to OpenAPI schema and served via the /schema endpoint. // Provider users can specify these fields in the DataStore's component CustomSpec. -type MongodCustomSpec struct{} +type MongodCustomSpec struct { + // SplitHorizonDNSRef is an optional reference to a pre-configured split horizon DNS configuration. + // When set, this component will be configured with split horizon DNS for multi-region deployments. + // +optional + SplitHorizonDNSRef *SplitHorizonDNSRef `json:"splitHorizonDNSRef,omitempty"` +} // ============================================================================= // MONGOS COMPONENT SPEC // ============================================================================= // MongosCustomSpec defines custom configuration for mongos (proxy) components. -type MongosCustomSpec struct{} +type MongosCustomSpec struct { + // SplitHorizonDNSRef is an optional reference to a pre-configured split horizon DNS configuration. + // When set, this component will be configured with split horizon DNS for multi-region deployments. + // +optional + SplitHorizonDNSRef *SplitHorizonDNSRef `json:"splitHorizonDNSRef,omitempty"` +} // ============================================================================= // PMM (MONITORING) COMPONENT SPEC @@ -83,9 +93,45 @@ type ShardedTopologyConfig struct { NumShards int32 `json:"numShards,omitempty"` } +// ============================================================================= +// SPLIT HORIZON DNS CONFIG (Pre-configured, referenced by DataStore) +// ============================================================================= + +// SplitHorizonDNSConfig defines DNS configuration for split horizon setup. +// This is typically stored in a ConfigMap in the cluster and referenced by DataStores. +type SplitHorizonDNSConfig struct { + // BaseDomainNameSuffix is the base domain name suffix for the cluster. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + BaseDomainNameSuffix string `json:"baseDomainNameSuffix"` + + // SecretName is the name of the secret containing CA certificate and key. + // +kubebuilder:validation:Required + SecretName string `json:"secretName"` +} + +// SplitHorizonDNSRef defines a reference to a pre-configured split horizon DNS configuration. +type SplitHorizonDNSRef struct { + // Name is the name of the split horizon configuration. + // The configuration should be stored as a ConfigMap in the cluster. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Namespace is the namespace where the split horizon configuration is stored. + // If omitted, defaults to the DataStore's namespace. + // +optional + Namespace string `json:"namespace,omitempty"` +} + // ============================================================================= // GLOBAL CONFIG // ============================================================================= // GlobalConfig defines global configuration that applies to the entire cluster. -type GlobalConfig struct{} +type GlobalConfig struct { + // SplitHorizonDNSRef is a reference to a pre-configured split horizon DNS configuration. + // The configuration should be stored as a ConfigMap in the cluster. + // +optional + SplitHorizonDNSRef *SplitHorizonDNSRef `json:"splitHorizonDNSRef,omitempty"` +} diff --git a/pkg/reconciler/provider.go b/pkg/reconciler/provider.go index 590c8a3..8baecea 100644 --- a/pkg/reconciler/provider.go +++ b/pkg/reconciler/provider.go @@ -7,6 +7,7 @@ import ( "github.com/openeverest/provider-sdk/pkg/apis/v2alpha1" "github.com/openeverest/provider-sdk/pkg/controller" "github.com/openeverest/provider-sdk/pkg/server" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -99,6 +100,11 @@ func newReconciler(p providerAdapter, opts ...ReconcilerOption) (*ProviderReconc } scheme := runtime.NewScheme() + // Register core Kubernetes types + if err := corev1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to add corev1 scheme: %w", err) + } + // Register core types if err := v2alpha1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("failed to add v2alpha1 scheme: %w", err) @@ -316,6 +322,12 @@ func (r *ProviderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return reconcile.Result{}, err } + // Refetch the DataStore to avoid conflicts from concurrent modifications + if err := r.Client.Get(ctx, req.NamespacedName, ds); err != nil { + logger.Error(err, "Failed to refetch DataStore before status update") + return reconcile.Result{}, client.IgnoreNotFound(err) + } + ds.Status = status.ToV2Alpha1() if err := r.Client.Status().Update(ctx, ds); err != nil { logger.Error(err, "Failed to update status")