From 1440398a7277fe24fc443cd8ba545d11947b132b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 14 Jul 2025 11:59:29 +0200 Subject: [PATCH 1/3] chore: Fix tests, partially by fixing reporting-task --- CHANGELOG.md | 2 +- rust/operator-binary/src/controller.rs | 2 +- rust/operator-binary/src/reporting_task/mod.rs | 4 ++-- rust/operator-binary/src/security/mod.rs | 2 +- rust/operator-binary/src/security/tls.rs | 7 ++++--- tests/templates/kuttl/smoke_v1/test_nifi_metrics.py | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac884ca..8bbb7cd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]). -- Added Listener support for NiFi ([#784]). +- Added Listener support for NiFi ([#784], [#XXX]). - Adds new telemetry CLI arguments and environment variables ([#782]). - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c16a591a..c956c651 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1349,7 +1349,7 @@ async fn build_node_rolegroup_statefulset( vec![&build_reporting_task_service_name(&nifi_cluster_name)], SecretFormat::TlsPkcs12, &requested_secret_lifetime, - LISTENER_VOLUME_NAME, + Some(LISTENER_VOLUME_NAME), ) .context(SecuritySnafu)?, ) diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index 1efd71b3..d2cc9360 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -51,7 +51,6 @@ use stackable_operator::{ use crate::{ controller::build_recommended_labels, crd::{APP_NAME, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, - listener::LISTENER_VOLUME_NAME, security::{ authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME}, build_tls_volume, @@ -357,7 +356,8 @@ fn build_reporting_task_job( // There is no correct way to configure this job since it's an implementation detail. // Also it will be dropped when support for 1.x is removed. &Duration::from_days_unchecked(1), - LISTENER_VOLUME_NAME, + // There is no listener volume we could get certs for + None, ) .context(SecretVolumeBuildFailureSnafu)?, ) diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs index bc304b30..f1fdcc0e 100644 --- a/rust/operator-binary/src/security/mod.rs +++ b/rust/operator-binary/src/security/mod.rs @@ -50,7 +50,7 @@ pub fn build_tls_volume( service_scopes: Vec<&str>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, - listener_scope: &str, + listener_scope: Option<&str>, ) -> Result { tls::build_tls_volume( nifi, diff --git a/rust/operator-binary/src/security/tls.rs b/rust/operator-binary/src/security/tls.rs index 932ab381..73413eaa 100644 --- a/rust/operator-binary/src/security/tls.rs +++ b/rust/operator-binary/src/security/tls.rs @@ -27,7 +27,7 @@ pub(crate) fn build_tls_volume( service_scopes: Vec<&str>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, - listener_scope: &str, + listener_scope: Option<&str>, ) -> Result { let mut secret_volume_source_builder = SecretOperatorVolumeSourceBuilder::new(nifi.server_tls_secret_class()); @@ -35,16 +35,17 @@ pub(crate) fn build_tls_volume( if secret_format == SecretFormat::TlsPkcs12 { secret_volume_source_builder.with_tls_pkcs12_password(STACKABLE_TLS_STORE_PASSWORD); } - for scope in service_scopes { secret_volume_source_builder.with_service_scope(scope); } + if let Some(listener_scope) = listener_scope { + secret_volume_source_builder.with_listener_volume_scope(listener_scope); + } Ok(VolumeBuilder::new(volume_name) .ephemeral( secret_volume_source_builder .with_pod_scope() - .with_listener_volume_scope(listener_scope) .with_format(secret_format) .with_auto_tls_cert_lifetime(*requested_secret_lifetime) .build() diff --git a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py index 336660c4..075e16e0 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py @@ -39,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://nifi-node-default-0.nifi-node-default-headless.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics/" # wait for 'timeout' seconds t_end = time.time() + timeout From bf44479961f6b59318946e967fc60d07a05ef826 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 14 Jul 2025 12:07:18 +0200 Subject: [PATCH 2/3] WIP --- tests/templates/kuttl/smoke_v1/test_nifi_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py index 075e16e0..86fd42a4 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py @@ -45,7 +45,7 @@ t_end = time.time() + timeout while time.time() < t_end: try: - response = requests.post(url) + response = requests.get(url) response.raise_for_status() if metric_name in response.text: print("Test metrics succeeded!") From ab659080a1ea010c3d114c16d50af01b198dddba Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 14 Jul 2025 13:53:18 +0200 Subject: [PATCH 3/3] fix iceberg test --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 3 +++ tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json | 2 +- .../templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json | 2 +- tests/templates/kuttl/iceberg/README.md | 2 +- tests/templates/kuttl/upgrade/test_nifi_metrics.py | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index e3e102ea..585df15a 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -95,6 +95,9 @@ stringData: ---- If you want to use a `ServiceMonitor` you'd need to configure it as follows: + +// TODO: The ServiceMonitor should be switched to the -metrics service + [source,yaml] ---- --- diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json index a5f773f1..bf689ddd 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json @@ -653,7 +653,7 @@ "properties": { "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml,/stackable/userdata/hive-config/hive-site.xml", - "hive-metastore-uri": "thrift://hive:9083", + "hive-metastore-uri": "thrift://hive-metastore:9083", "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json index c63158b9..1a3d6c88 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json @@ -593,7 +593,7 @@ "properties": { "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml", - "hive-metastore-uri": "thrift://hive:9083", + "hive-metastore-uri": "thrift://hive-metastore:9083", "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md index b856148c..ce884d9e 100644 --- a/tests/templates/kuttl/iceberg/README.md +++ b/tests/templates/kuttl/iceberg/README.md @@ -25,7 +25,7 @@ index 09783fa..23c679f 100644 @@ -483,7 +485,7 @@ "properties": { "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", - "hive-metastore-uri": "thrift://hive:9083", + "hive-metastore-uri": "thrift://hive-metastore:9083", - "s3-endpoint": "https://minio.kuttl-test-patient-tarpon.svc.cluster.local:9000", + "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", diff --git a/tests/templates/kuttl/upgrade/test_nifi_metrics.py b/tests/templates/kuttl/upgrade/test_nifi_metrics.py index f0f00d0c..541abe91 100755 --- a/tests/templates/kuttl/upgrade/test_nifi_metrics.py +++ b/tests/templates/kuttl/upgrade/test_nifi_metrics.py @@ -39,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://test-nifi-node-default-0.test-nifi-node-default-headless.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://test-nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout