diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 223bc9c..effaa8f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -8,6 +8,7 @@ rules: - "" resources: - configmaps + - secrets - services verbs: - create diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 62b8d67..7d464e3 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,5 @@ ## Append samples of your project ## resources: +- sample_config.yaml - v1alpha1_valkeycluster.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/sample_config.yaml b/config/samples/sample_config.yaml new file mode 100644 index 0000000..f3d86c6 --- /dev/null +++ b/config/samples/sample_config.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: valkeycluster-sample-conf +data: + extra.conf: | + maxmemory 50mb + maxmemory-policy allkeys-lfu + aof.conf: | + appendonly yes + appendfilename "orbit.aof" diff --git a/internal/controller/deployment.go b/internal/controller/deployment.go index ab5729e..3587fb6 100644 --- a/internal/controller/deployment.go +++ b/internal/controller/deployment.go @@ -23,7 +23,7 @@ import ( valkeyiov1alpha1 "valkey.io/valkey-operator/api/v1alpha1" ) -func createClusterDeployment(cluster *valkeyiov1alpha1.ValkeyCluster) *appsv1.Deployment { +func createClusterDeployment(cluster *valkeyiov1alpha1.ValkeyCluster, configVolumes []corev1.Volume) *appsv1.Deployment { image := DefaultImage if cluster.Spec.Image != "" { image = cluster.Spec.Image @@ -105,32 +105,15 @@ func createClusterDeployment(cluster *valkeyiov1alpha1.ValkeyCluster) *appsv1.De MountPath: "/config", ReadOnly: true, }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "scripts", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster.Name, - }, - DefaultMode: func(i int32) *int32 { return &i }(0755), - }, - }, - }, - { - Name: "valkey-conf", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster.Name, - }, + { + Name: "user-conf", + MountPath: "/config/valkey.conf.d", + ReadOnly: true, }, }, }, }, + Volumes: configVolumes, }, }, }, diff --git a/internal/controller/deployment_test.go b/internal/controller/deployment_test.go index e782b48..acc7d1f 100644 --- a/internal/controller/deployment_test.go +++ b/internal/controller/deployment_test.go @@ -19,6 +19,7 @@ package controller import ( "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" valkeyv1 "valkey.io/valkey-operator/api/v1alpha1" ) @@ -33,7 +34,9 @@ func TestCreateClusterDeployment(t *testing.T) { }, } - d := createClusterDeployment(cluster) + volumes := []corev1.Volume{getConfigVolume("mycluster", "myvol", false)} + + d := createClusterDeployment(cluster, volumes) if d.Name != "" { t.Errorf("Expected empty name field, got %v", d.Name) @@ -50,4 +53,9 @@ func TestCreateClusterDeployment(t *testing.T) { if d.Spec.Template.Spec.Containers[0].Image != "container:version" { t.Errorf("Expected %v, got %v", "container:version", d.Spec.Template.Spec.Containers[0].Image) } + + // Volume check + if volLen := len(d.Spec.Template.Spec.Volumes); volLen != 1 { + t.Errorf("Expected %v volume, got %v", 1, volLen) + } } diff --git a/internal/controller/valkeycluster_controller.go b/internal/controller/valkeycluster_controller.go index 4d5c78f..365a4b6 100644 --- a/internal/controller/valkeycluster_controller.go +++ b/internal/controller/valkeycluster_controller.go @@ -63,6 +63,7 @@ var scripts embed.FS // +kubebuilder:rbac:groups=valkey.io,resources=valkeyclusters/finalizers,verbs=update // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch @@ -87,7 +88,7 @@ func (r *ValkeyClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, err } - if err := r.upsertConfigMap(ctx, cluster); err != nil { + if err := r.upsertDefaultConfigMap(ctx, cluster); err != nil { setCondition(cluster, valkeyiov1alpha1.ConditionReady, valkeyiov1alpha1.ReasonConfigMapError, err.Error(), metav1.ConditionFalse) _ = r.updateStatus(ctx, cluster, nil) return ctrl.Result{}, err @@ -225,7 +226,7 @@ func (r *ValkeyClusterReconciler) upsertService(ctx context.Context, cluster *va } // Create or update a basic valkey.conf -func (r *ValkeyClusterReconciler) upsertConfigMap(ctx context.Context, cluster *valkeyiov1alpha1.ValkeyCluster) error { +func (r *ValkeyClusterReconciler) upsertDefaultConfigMap(ctx context.Context, cluster *valkeyiov1alpha1.ValkeyCluster) error { readiness, err := scripts.ReadFile("scripts/readiness-check.sh") if err != nil { return err @@ -247,7 +248,8 @@ func (r *ValkeyClusterReconciler) upsertConfigMap(ctx context.Context, cluster * "valkey.conf": ` cluster-enabled yes protected-mode no -cluster-node-timeout 2000`, +cluster-node-timeout 2000 +include /config/valkey.conf.d/*.conf`, }, } if err := controllerutil.SetControllerReference(cluster, cm, r.Scheme); err != nil { @@ -281,9 +283,15 @@ func (r *ValkeyClusterReconciler) upsertDeployments(ctx context.Context, cluster expected := int(cluster.Spec.Shards * (1 + cluster.Spec.Replicas)) + // Get script volume, default config, and user config volumes + configVolumes, err := getConfigVolumes(ctx, r.Client, cluster) + if err != nil { + return err + } + // Create missing deployments for i := len(existing.Items); i < expected; i++ { - deployment := createClusterDeployment(cluster) + deployment := createClusterDeployment(cluster, configVolumes) if err := controllerutil.SetControllerReference(cluster, deployment, r.Scheme); err != nil { return err } diff --git a/internal/controller/volumes.go b/internal/controller/volumes.go new file mode 100644 index 0000000..2f6712a --- /dev/null +++ b/internal/controller/volumes.go @@ -0,0 +1,131 @@ +/* +Copyright 2025 Valkey Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + valkeyiov1alpha1 "valkey.io/valkey-operator/api/v1alpha1" +) + +func getConfigVolumes(ctx context.Context, c client.Client, cluster *valkeyiov1alpha1.ValkeyCluster) ([]corev1.Volume, error) { + + // Volume containing health, and liveliness scripts + scriptsVolume := corev1.Volume{ + Name: "scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cluster.Name, + }, + DefaultMode: func(i int32) *int32 { return &i }(0755), + }, + }, + } + + // Volume containing default Valkey configuration + defaultConfigVolume := corev1.Volume{ + Name: "valkey-conf", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cluster.Name, + }, + }, + }, + } + + // User config volume + userConfigVolume, err := getUserConfigVolume(ctx, c, cluster) + if err != nil { + return nil, err + } + + return []corev1.Volume{ + scriptsVolume, + defaultConfigVolume, + userConfigVolume, + }, nil +} + +// Discover user-created Valkey configuration. This can be either a Secret, or ConfigMap, created by the user +func getUserConfigVolume(ctx context.Context, c client.Client, cluster *valkeyiov1alpha1.ValkeyCluster) (corev1.Volume, error) { + log := logf.FromContext(ctx) + + configMapName := cluster.Name + "-conf" + userConfigFilter := types.NamespacedName{ + Namespace: cluster.Namespace, + Name: configMapName, + } + + // First, look for a Secret named "$clusterName-conf" + err := c.Get(ctx, userConfigFilter, &corev1.Secret{}) + if err == nil { + log.V(1).Info("found user-created secret config") + return getSecretConfigVolume("user-conf", configMapName, false), nil + } + if !apierrors.IsNotFound(err) { + log.Error(err, "failed to search for user config Secret") + return corev1.Volume{}, err + } + + // Next, look for a ConfigMap named "$clusterName-conf" + err = c.Get(ctx, userConfigFilter, &corev1.ConfigMap{}) + if err == nil { + log.V(1).Info("found user-created configMap") + return getConfigVolume("user-conf", configMapName, false), nil + } + if !apierrors.IsNotFound(err) { + log.Error(err, "failed to search for user config ConfigMap") + return corev1.Volume{}, err + } + + // If neither found, create empty (optional) config-volume + log.V(1).Info("using empty configMap") + return getConfigVolume("user-conf", configMapName, true), nil +} + +func getConfigVolume(configVolumeName, configMapName string, optional bool) corev1.Volume { + return corev1.Volume{ + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + Optional: func(b bool) *bool { return &b }(optional), + }, + }, + } +} + +func getSecretConfigVolume(configVolumeName, configMapName string, optional bool) corev1.Volume { + return corev1.Volume{ + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: configMapName, + Optional: func(b bool) *bool { return &b }(optional), + }, + }, + } +} diff --git a/internal/controller/volumes_test.go b/internal/controller/volumes_test.go new file mode 100644 index 0000000..b44e095 --- /dev/null +++ b/internal/controller/volumes_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2025 Valkey Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestGetConfigVolume(t *testing.T) { + + volumeName := "test1" + mapName := "alice-conf" + optional := false + + vol := getConfigVolume(volumeName, mapName, optional) + + if vol.Name != volumeName { + t.Errorf("Expected volume name to be '%v', got %v", volumeName, vol.Name) + } + if reflect.TypeOf(vol.VolumeSource) != reflect.TypeOf(corev1.VolumeSource{}) { + t.Errorf("Expected VolumeSource to be native k8s type") + } + if reflect.TypeOf(vol.VolumeSource.ConfigMap) != reflect.TypeOf(&corev1.ConfigMapVolumeSource{}) { + t.Errorf("Expected VolumeSource.ConfigMap to be native k8s type") + } + if vol.VolumeSource.ConfigMap.Name != mapName { + t.Errorf("Expected ConfigMap name to be '%v', got '%v'", mapName, vol.VolumeSource.ConfigMap.Name) + } + if *vol.VolumeSource.ConfigMap.Optional != optional { + t.Errorf("Expection ConfigMap.Optional to be %v, got %v", optional, vol.VolumeSource.ConfigMap.Optional) + } +} + +func TestGetSecretConfigVolume(t *testing.T) { + + volumeName := "test2" + secretName := "bob-conf" + optional := true + + vol := getSecretConfigVolume(volumeName, secretName, optional) + + if vol.Name != volumeName { + t.Errorf("Expected volume name to be '%v', got %v", volumeName, vol.Name) + } + if reflect.TypeOf(vol.VolumeSource) != reflect.TypeOf(corev1.VolumeSource{}) { + t.Errorf("Expected VolumeSource to be native k8s type") + } + if reflect.TypeOf(vol.VolumeSource.Secret) != reflect.TypeOf(&corev1.SecretVolumeSource{}) { + t.Errorf("Expected VolumeSource.Secret to be native k8s type") + } + if vol.VolumeSource.Secret.SecretName != secretName { + t.Errorf("Expected Secret name to be '%v', got '%v'", secretName, vol.VolumeSource.Secret.SecretName) + } + if *vol.VolumeSource.Secret.Optional != optional { + t.Errorf("Expection Secret.Optional to be %v, got %v", optional, vol.VolumeSource.Secret.Optional) + } +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index b6f8d16..81cbe3f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -283,7 +283,12 @@ var _ = Describe("Manager", Ordered, func() { Context("when a ValkeyCluster CR is applied", func() { It("creates a Valkey Cluster deployment", func() { - By("creating the CR") + By("creating a custom configMap") + cmd := exec.Command("kubectl", "create", "-f", "config/samples/sample_config.yaml") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ValkeyCluster custom configMap") + + By("creating the ValkeyCluster CR") cmd := exec.Command("kubectl", "create", "-f", "config/samples/v1alpha1_valkeycluster.yaml") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to create ValkeyCluster CR") @@ -313,7 +318,7 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyServiceExists).Should(Succeed()) - By("validating the ConfigMap") + By("validating the default ConfigMap") verifyConfigMapExists := func(g Gomega) { cmd := exec.Command("kubectl", "get", "configmap", valkeyClusterName) _, err := utils.Run(cmd) @@ -321,6 +326,14 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyConfigMapExists).Should(Succeed()) + By("validating the custom ConfigMap") + verifyConfigMapExists = func(g Gomega) { + cmd := exec.Command("kubectl", "get", "configmap", valkeyClusterName + "-conf") + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyConfigMapExists).Should(Succeed()) + By("validating Deployments") verifyDeploymentsExists := func(g Gomega) { cmd := exec.Command("kubectl", "get", "deployments", @@ -494,6 +507,14 @@ var _ = Describe("Manager", Ordered, func() { _, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) + // Verify user configMap took effect. The sample config sets maxmemory to 50Mb. + cmd := exec.Command("kubectl", "run", "client2", + fmt.Sprintf("--image=%s", valkeyClientImage), "--restart=Never", "--", + "valkey-cli", "-c", "-h", clusterFqdn, "CONFIG", "GET", "maxmemory") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to verify sample config") + g.Expect(output).To(ContainSubstring("52428800")) + cmd = exec.Command("kubectl", "wait", "pod/client", "--for=jsonpath={.status.phase}=Succeeded", "--timeout=30s") _, err = utils.Run(cmd) @@ -508,6 +529,11 @@ var _ = Describe("Manager", Ordered, func() { _, err = utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) + cmd = exec.Command("kubectl", "delete", "pod", "client2", + "--wait=true", "--timeout=30s") + _, err = utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + // The cluster should be ok. g.Expect(output).To(ContainSubstring("cluster_state:ok")) } @@ -539,7 +565,7 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyServiceRemoved).Should(Succeed()) - By("validating that the ConfigMap does not exist") + By("validating that the default ConfigMap does not exist") verifyConfigMapRemoved := func(g Gomega) { cmd := exec.Command("kubectl", "get", "configmap", valkeyClusterName) _, err := utils.Run(cmd) @@ -547,6 +573,14 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyConfigMapRemoved).Should(Succeed()) + By("validating that the user ConfigMap remains") + verifyConfigMapRemains := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "configmap", valkeyClusterName + "-conf") + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyConfigMapRemains).Should(Succeed()) + By("validating that no Deployment exist") verifyDeploymentsRemoved := func(g Gomega) { cmd := exec.Command("kubectl", "get", "deployments",