diff --git a/Makefile b/Makefile index d993d4b..b719c39 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,11 @@ CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen IMG_PREFIX ?= controller IMG_TAG ?= latest -# ENABLE_METRICS: If set to true, includes Prometheus Service and ServiceMonitor resources. +# ENABLE_METRICS: If set to true, includes Prometheus Service resources. ENABLE_METRICS ?= false -# ENABLE_TLS: If set to true (and ENABLE_METRICS is true), configures metrics to use HTTPS with CertManager. ENABLE_TLS ?= false +# ENABLE_WEBHOOK: If set to true, includes validating webhook. Requires ENABLE_TLS=true. +ENABLE_WEBHOOK ?= false # Default value for ignore-not-found flag in undeploy target ignore-not-found ?= true @@ -235,21 +236,30 @@ endif # Temporary directory for building manifests BUILD_DIR := $(ROOT_DIR)/bin/build -# Internal target to build manifests in a temporary directory to keep the source config clean. -# This prevents 'kustomize edit' from modifying your local git state. -# Features (Metrics, TLS) are enabled by adding Kustomize Components to the temporary copy. -# TODO: we can do better for prometheus metrics ports that are added by manager_prometheus_metrics.yaml +# Build manifests in a temp directory to keep source config clean. +# Features are enabled by adding Kustomize Components. .PHONY: build-manifests-temp build-manifests-temp: manifests $(KUSTOMIZE) @mkdir -p $(BUILD_DIR) @rm -rf $(BUILD_DIR)/config @cp -r config $(BUILD_DIR)/ @cd $(BUILD_DIR)/config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG} + @# TLS: Add certmanager component for certificates + @if [ "$(ENABLE_TLS)" = "true" ]; then \ + cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../certmanager; \ + fi + @# Webhook: Requires TLS for certificates + @if [ "$(ENABLE_WEBHOOK)" = "true" ]; then \ + if [ "$(ENABLE_TLS)" != "true" ]; then \ + echo "ERROR: ENABLE_WEBHOOK=true requires ENABLE_TLS=true"; exit 1; \ + fi; \ + cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../webhook; \ + fi + @# Metrics: Add prometheus, with TLS config if enabled @if [ "$(ENABLE_METRICS)" = "true" ]; then \ cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../prometheus; \ if [ "$(ENABLE_TLS)" = "true" ]; then \ - cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../certmanager && \ - $(KUSTOMIZE) edit add component ../prometheus/tls; \ + cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../prometheus/tls; \ else \ cd $(BUILD_DIR)/config/prometheus && $(KUSTOMIZE) edit add patch --path manager_prometheus_metrics.yaml --kind Deployment --name controller-manager; \ fi; \ @@ -278,21 +288,52 @@ undeploy: build-manifests-temp ## Undeploy controller from the K8s cluster. Use .PHONY: deploy-with-metrics deploy-with-metrics: ENABLE_METRICS=true -deploy-with-metrics: deploy ## Deploy with metrics enabled. +deploy-with-metrics: deploy ## Deploy with metrics (HTTP). .PHONY: undeploy-with-metrics undeploy-with-metrics: ENABLE_METRICS=true -undeploy-with-metrics: undeploy ## Undeploy with metrics enabled. - -.PHONY: deploy-with-metrics-tls-enabled -deploy-with-metrics-tls-enabled: ENABLE_TLS=true -deploy-with-metrics-tls-enabled: ENABLE_METRICS=true -deploy-with-metrics-tls-enabled: deploy ## Deploy with metrics and TLS enabled. - -.PHONY: undeploy-with-metrics-tls-enabled -undeploy-with-metrics-tls-enabled: ENABLE_TLS=true -undeploy-with-metrics-tls-enabled: ENABLE_METRICS=true -undeploy-with-metrics-tls-enabled: undeploy ## Undeploy with metrics and TLS enabled. +undeploy-with-metrics: undeploy ## Undeploy with metrics. + +.PHONY: deploy-with-metrics-and-tls +deploy-with-metrics-and-tls: ENABLE_METRICS=true +deploy-with-metrics-and-tls: ENABLE_TLS=true +deploy-with-metrics-and-tls: deploy ## Deploy with metrics and TLS. + +.PHONY: undeploy-with-metrics-and-tls +undeploy-with-metrics-and-tls: ENABLE_METRICS=true +undeploy-with-metrics-and-tls: ENABLE_TLS=true +undeploy-with-metrics-and-tls: undeploy ## Undeploy with metrics and TLS. + +.PHONY: deploy-with-tls +deploy-with-tls: ENABLE_TLS=true +deploy-with-tls: deploy ## Deploy with TLS (cert-manager). + +.PHONY: undeploy-with-tls +undeploy-with-tls: ENABLE_TLS=true +undeploy-with-tls: undeploy ## Undeploy with TLS. + +.PHONY: deploy-with-webhook +deploy-with-webhook: ENABLE_TLS=true +deploy-with-webhook: ENABLE_WEBHOOK=true +deploy-with-webhook: deploy ## Deploy with webhook (includes TLS). + +.PHONY: undeploy-with-webhook +undeploy-with-webhook: ENABLE_TLS=true +undeploy-with-webhook: ENABLE_WEBHOOK=true +undeploy-with-webhook: undeploy ## Undeploy with webhook. + +# Deploy with all features: metrics, TLS, webhook. +.PHONY: deploy-full +deploy-full: ENABLE_METRICS=true +deploy-full: ENABLE_TLS=true +deploy-full: ENABLE_WEBHOOK=true +deploy-full: deploy ## Deploy with all features: metrics, TLS, webhook. + +.PHONY: undeploy-full +undeploy-full: ENABLE_METRICS=true +undeploy-full: ENABLE_TLS=true +undeploy-full: ENABLE_WEBHOOK=true +undeploy-full: undeploy ## Undeploy with all features. ## -------------------------------------- ## Testing diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml index 18dac92..ccea49c 100644 --- a/config/certmanager/certificate.yaml +++ b/config/certmanager/certificate.yaml @@ -8,17 +8,3 @@ metadata: spec: selfSigned: {} --- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: metrics-certs - namespace: system -spec: - commonName: nrr-metrics - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: metrics-server-cert diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml deleted file mode 100644 index 5fac35d..0000000 --- a/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - --leader-elect - - --health-probe-bind-address=:8081 - - --enable-webhook=true - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: webhook-server-certs \ No newline at end of file diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 3fa4932..13b50a7 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -74,7 +74,6 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 - - --enable-webhook=false image: controller:latest imagePullPolicy: IfNotPresent name: manager diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index fa8f9ea..12dd95a 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,22 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: -- monitor.yaml - metrics_service.yaml -patches: -# Bind metrics to port 8080 for HTTP. -# This matches the Service and ServiceMonitor configuration in this directory. -# - path: manager_prometheus_metrics.yaml -# target: -# kind: Deployment -# name: controller-manager - -# By default, metrics are disabled in the manager (default : "0"). -# This component adds the Service and ServiceMonitor for Prometheus, -# and applies the patch to bind the manager to port :8080(it is done in Makefile for now). - -# Patches for TLS are in the 'tls' component which will: -# 1. Overlay the HTTPS args (:8443) and security flags -# 2. Add ServiceMonitor TLS config -# 3. Mount CertManager secrets diff --git a/config/prometheus/metrics_service.yaml b/config/prometheus/metrics_service.yaml index 220341a..4b64ed7 100644 --- a/config/prometheus/metrics_service.yaml +++ b/config/prometheus/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: nrrcontroller app.kubernetes.io/managed-by: kustomize - name: controller-manager-metrics-service + name: metrics-service namespace: system spec: ports: diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml deleted file mode 100644 index 0cd4f99..0000000 --- a/config/prometheus/monitor.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: controller-manager - app.kubernetes.io/name: nrrcontroller - app.kubernetes.io/managed-by: kustomize - name: controller-manager-metrics-monitor - namespace: system -spec: - endpoints: - - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification, exposing the system to potential man-in-the-middle attacks. - # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. - # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, - # which securely references the certificate from the 'metrics-server-cert' secret. - insecureSkipVerify: false - selector: - matchLabels: - control-plane: controller-manager - app.kubernetes.io/name: nrrcontroller diff --git a/config/prometheus/tls/certificate.yaml b/config/prometheus/tls/certificate.yaml new file mode 100644 index 0000000..e615ac4 --- /dev/null +++ b/config/prometheus/tls/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: metrics-certs + namespace: system +spec: + dnsNames: + - nrr-metrics-service.nrr-system.svc + - nrr-metrics-service.nrr-system.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert diff --git a/config/prometheus/tls/kustomization.yaml b/config/prometheus/tls/kustomization.yaml index c162ce7..0f39bb5 100644 --- a/config/prometheus/tls/kustomization.yaml +++ b/config/prometheus/tls/kustomization.yaml @@ -1,15 +1,14 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component +resources: +- certificate.yaml + patches: # Enable HTTPS args in Deployment - path: manager_prometheus_metrics_tls.yaml target: kind: Deployment -# Configure ServiceMonitor for TLS -- path: monitor_tls_patch.yaml - target: - kind: ServiceMonitor # Mount CertManager secrets in Deployment - path: cert_metrics_manager_patch.yaml target: @@ -18,63 +17,3 @@ patches: - path: metrics_service_tls_patch.yaml target: kind: Service - -replacements: - - source: - kind: Service - version: v1 - name: controller-manager-metrics-service - fieldPath: metadata.name - targets: - - select: - kind: Certificate - group: cert-manager.io - version: v1 - name: metrics-certs - fieldPaths: - - spec.dnsNames.0 - - spec.dnsNames.1 - options: - delimiter: '.' - index: 0 - create: true - - select: - kind: ServiceMonitor - group: monitoring.coreos.com - version: v1 - name: controller-manager-metrics-monitor - fieldPaths: - - spec.endpoints.0.tlsConfig.serverName - options: - delimiter: '.' - index: 0 - create: true - - source: - kind: Service - version: v1 - name: controller-manager-metrics-service - fieldPath: metadata.namespace - targets: - - select: - kind: Certificate - group: cert-manager.io - version: v1 - name: metrics-certs - fieldPaths: - - spec.dnsNames.0 - - spec.dnsNames.1 - options: - delimiter: '.' - index: 1 - create: true - - select: - kind: ServiceMonitor - group: monitoring.coreos.com - version: v1 - name: controller-manager-metrics-monitor - fieldPaths: - - spec.endpoints.0.tlsConfig.serverName - options: - delimiter: '.' - index: 1 - create: true diff --git a/config/prometheus/tls/metrics_service_tls_patch.yaml b/config/prometheus/tls/metrics_service_tls_patch.yaml index 03d8a7d..f8d0eeb 100644 --- a/config/prometheus/tls/metrics_service_tls_patch.yaml +++ b/config/prometheus/tls/metrics_service_tls_patch.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: controller-manager-metrics-service + name: metrics-service namespace: system spec: ports: diff --git a/config/prometheus/tls/monitor_tls_patch.yaml b/config/prometheus/tls/monitor_tls_patch.yaml deleted file mode 100644 index 02da4bb..0000000 --- a/config/prometheus/tls/monitor_tls_patch.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Patch for Prometheus ServiceMonitor to enable secure TLS configuration -# using certificates managed by cert-manager -- op: replace - path: /spec/endpoints/0/tlsConfig - value: - # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize - serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc - insecureSkipVerify: false - ca: - secret: - name: metrics-server-cert - key: ca.crt - cert: - secret: - name: metrics-server-cert - key: tls.crt - keySecret: - name: metrics-server-cert - key: tls.key -- op: replace - path: /spec/endpoints/0/port - value: https -- op: replace - path: /spec/endpoints/0/scheme - value: https diff --git a/config/webhook/certificate.yaml b/config/webhook/certificate.yaml new file mode 100644 index 0000000..17aaa5e --- /dev/null +++ b/config/webhook/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert + namespace: system +spec: + dnsNames: + - nrr-webhook-service.nrr-system.svc + - nrr-webhook-service.nrr-system.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-certs diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 4aba990..9f7d302 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,9 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + resources: - manifests.yaml - service.yaml +- certificate.yaml configurations: - kustomizeconfig.yaml -patchesStrategicMerge: -- webhook_patch.yaml \ No newline at end of file +patches: +- path: webhook_patch.yaml + target: + kind: ValidatingWebhookConfiguration +- path: manager_webhook_patch.yaml + target: + group: apps + version: v1 + kind: Deployment + name: controller-manager + namespace: system diff --git a/config/webhook/manager_webhook_patch.yaml b/config/webhook/manager_webhook_patch.yaml new file mode 100644 index 0000000..2c26eb4 --- /dev/null +++ b/config/webhook/manager_webhook_patch.yaml @@ -0,0 +1,29 @@ +# Enable webhook +- op: add + path: /spec/template/spec/containers/0/args/- + value: --enable-webhook=true + +# Add webhook port +- op: add + path: /spec/template/spec/containers/0/ports/- + value: + containerPort: 9443 + name: webhook-server + protocol: TCP + +# Add volume mount +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + +# Add volume +- op: add + path: /spec/template/spec/volumes/- + value: + name: cert + secret: + defaultMode: 420 + secretName: webhook-server-certs diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 3d7634b..2917855 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -10,7 +10,7 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-nodereadiness-io-v1alpha1-nodereadinessrule + path: /validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule failurePolicy: Fail name: vnodereadinessrule.kb.io rules: diff --git a/config/webhook/webhook_patch.yaml b/config/webhook/webhook_patch.yaml index 9ac5d73..4e64e25 100644 --- a/config/webhook/webhook_patch.yaml +++ b/config/webhook/webhook_patch.yaml @@ -3,11 +3,10 @@ kind: ValidatingWebhookConfiguration metadata: name: validating-webhook-configuration annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + cert-manager.io/inject-ca-from: nrr-system/nrr-serving-cert webhooks: - name: vnodereadinessrule.kb.io clientConfig: service: name: webhook-service namespace: system - path: /validate-nodereadiness-io-v1alpha1-nodereadinessrule diff --git a/internal/webhook/nodereadinessgaterule_webhook.go b/internal/webhook/nodereadinessgaterule_webhook.go index e4a5015..421fea2 100644 --- a/internal/webhook/nodereadinessgaterule_webhook.go +++ b/internal/webhook/nodereadinessgaterule_webhook.go @@ -43,7 +43,7 @@ func NewNodeReadinessRuleWebhook(c client.Client) *NodeReadinessRuleWebhook { } } -// +kubebuilder:webhook:path=/validate-nodereadiness-io-v1alpha1-nodereadinessrule,mutating=false,failurePolicy=fail,sideEffects=None,groups=readiness.node.x-k8s.io,resources=nodereadinessrules,verbs=create;update,versions=v1alpha1,name=vnodereadinessrule.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-readiness-node-x-k8s-io-v1alpha1-nodereadinessrule,mutating=false,failurePolicy=fail,sideEffects=None,groups=readiness.node.x-k8s.io,resources=nodereadinessrules,verbs=create;update,versions=v1alpha1,name=vnodereadinessrule.kb.io,admissionReviewVersions=v1 // validateNodeReadinessRule performs validation logic. func (w *NodeReadinessRuleWebhook) validateNodeReadinessRule(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule, isUpdate bool) field.ErrorList {