From 6f186a03aa77bc1274c7e8a9a48b9e59300fa942 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 28 Apr 2025 08:42:15 +0200 Subject: [PATCH 01/13] Clean up smoke test --- tests/templates/kuttl/smoke/20-assert.yaml | 2 +- ...k.yaml.j2 => 20-install-zookeeper.yaml.j2} | 10 +-- tests/templates/kuttl/smoke/30-assert.yaml | 4 +- .../kuttl/smoke/30-install-nifi.yaml.j2 | 64 +++++++++++-------- tests/templates/kuttl/smoke/31-assert.yaml.j2 | 2 +- tests/templates/kuttl/smoke/32-assert.yaml | 12 ++-- tests/templates/kuttl/smoke/33-assert.yaml | 2 +- tests/templates/kuttl/smoke/40-assert.yaml | 2 +- .../kuttl/smoke/40-scale-up-nifi.yaml.j2 | 2 +- tests/templates/kuttl/smoke/70-assert.yaml | 42 ------------ .../kuttl/smoke/70-enable-anonymous.yaml.j2 | 9 --- tests/templates/kuttl/smoke/test_nifi.py | 2 +- .../kuttl/smoke/test_nifi_metrics.py | 2 +- tests/test-definition.yaml | 10 +-- 14 files changed, 54 insertions(+), 111 deletions(-) rename tests/templates/kuttl/smoke/{20-install-zk.yaml.j2 => 20-install-zookeeper.yaml.j2} (78%) delete mode 100644 tests/templates/kuttl/smoke/70-assert.yaml delete mode 100644 tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml index e0766c49..49ba7437 100644 --- a/tests/templates/kuttl/smoke/20-assert.yaml +++ b/tests/templates/kuttl/smoke/20-assert.yaml @@ -6,7 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-zk-server-default + name: zookeeper-server-default status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/smoke/20-install-zk.yaml.j2 b/tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2 similarity index 78% rename from tests/templates/kuttl/smoke/20-install-zk.yaml.j2 rename to tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2 index 100927b1..103845c4 100644 --- a/tests/templates/kuttl/smoke/20-install-zk.yaml.j2 +++ b/tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperCluster metadata: - name: test-zk + name: zookeeper spec: image: productVersion: "{{ test_scenario['values']['zookeeper'] }}" @@ -19,11 +19,3 @@ spec: roleGroups: default: replicas: 1 ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperZnode -metadata: - name: test-nifi-znode -spec: - clusterRef: - name: test-zk diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml index ae825d11..d075654f 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -6,7 +6,7 @@ timeout: 1200 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-default + name: nifi-node-default spec: template: spec: @@ -18,7 +18,7 @@ status: apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: test-nifi-node + name: nifi-node status: expectedPods: 2 currentHealthy: 2 diff --git a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 index aceebaf6..6d0940e6 100644 --- a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 @@ -1,33 +1,8 @@ --- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: simple-nifi-users -spec: - provider: - static: - userCredentialsSecret: - name: simple-nifi-admin-credentials ---- -apiVersion: v1 -kind: Secret -metadata: - name: simple-nifi-admin-credentials -stringData: - admin: > - passwordWithSpecialCharacter\@<&>"' ---- -apiVersion: v1 -kind: Secret -metadata: - name: nifi-sensitive-property-key -stringData: - nifiSensitivePropsKey: mYsUp3rS3cr3tk3y ---- apiVersion: nifi.stackable.tech/v1alpha1 kind: NifiCluster metadata: - name: test-nifi + name: nifi spec: image: {% if test_scenario['values']['nifi'].find(",") > 0 %} @@ -39,10 +14,10 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - zookeeperConfigMapName: test-nifi-znode + zookeeperConfigMapName: nifi-znode listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - - authenticationClass: simple-nifi-users + - authenticationClass: nifi-users hostHeaderCheck: allowAll: false additionalAllowedHosts: @@ -73,3 +48,36 @@ spec: "nifi.properties": "nifi.diagnostics.on.shutdown.enabled": "false" "nifi.diagnostics.on.shutdown.max.filecount": "20" +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-admin-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-admin-credentials +stringData: + admin: > + passwordWithSpecialCharacter\@<&>"' +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-znode +spec: + clusterRef: + name: zookeeper diff --git a/tests/templates/kuttl/smoke/31-assert.yaml.j2 b/tests/templates/kuttl/smoke/31-assert.yaml.j2 index 06b1dc78..7b2eacd4 100644 --- a/tests/templates/kuttl/smoke/31-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/31-assert.yaml.j2 @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 30 commands: -- script: kubectl get cm -n $NAMESPACE test-nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z +- script: kubectl get cm -n $NAMESPACE nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml index 00da1613..960b4789 100644 --- a/tests/templates/kuttl/smoke/32-assert.yaml +++ b/tests/templates/kuttl/smoke/32-assert.yaml @@ -7,13 +7,13 @@ commands: # Test envOverrides # - script: | - kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' - kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' - kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' + kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' # # Test configOverrides # - script: | - kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false" - kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false" - kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20" + kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false" + kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false" + kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20" diff --git a/tests/templates/kuttl/smoke/33-assert.yaml b/tests/templates/kuttl/smoke/33-assert.yaml index b2a98140..271a8bec 100644 --- a/tests/templates/kuttl/smoke/33-assert.yaml +++ b/tests/templates/kuttl/smoke/33-assert.yaml @@ -4,4 +4,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl exec -n $NAMESPACE --container nifi test-nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + - script: kubectl exec -n $NAMESPACE --container nifi nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/40-assert.yaml b/tests/templates/kuttl/smoke/40-assert.yaml index 88f50b77..c3bcd76b 100644 --- a/tests/templates/kuttl/smoke/40-assert.yaml +++ b/tests/templates/kuttl/smoke/40-assert.yaml @@ -6,7 +6,7 @@ timeout: 1200 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-default + name: nifi-node-default status: readyReplicas: 3 replicas: 3 diff --git a/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 b/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 index 987b0745..37ded261 100644 --- a/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 @@ -4,5 +4,5 @@ kind: TestStep commands: - script: >- kubectl --namespace $NAMESPACE - patch nificlusters.nifi.stackable.tech test-nifi + patch nificlusters.nifi.stackable.tech nifi --type=merge --patch '{"spec":{"nodes": {"roleGroups": {"default": {"replicas": 3}}}}}' diff --git a/tests/templates/kuttl/smoke/70-assert.yaml b/tests/templates/kuttl/smoke/70-assert.yaml deleted file mode 100644 index 29b0ff4d..00000000 --- a/tests/templates/kuttl/smoke/70-assert.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: v1 -kind: Event -reason: Started -source: - component: kubelet -involvedObject: - apiVersion: v1 - kind: Pod - name: test-nifi-node-default-0 ---- -apiVersion: v1 -kind: Event -reason: Started -source: - component: kubelet -involvedObject: - apiVersion: v1 - kind: Pod - name: test-nifi-node-default-1 ---- -apiVersion: v1 -kind: Event -reason: Started -source: - component: kubelet -involvedObject: - apiVersion: v1 - kind: Pod - name: test-nifi-node-default-2 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-nifi-node-default -status: - readyReplicas: 3 - replicas: 3 diff --git a/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 b/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 deleted file mode 100644 index f39ce021..00000000 --- a/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: >- - kubectl --namespace $NAMESPACE - patch nificlusters.nifi.stackable.tech test-nifi - --type=merge --patch '{"spec":{"config": {"authentication": {"allowAnonymousAccess": true}}}}' - - command: kubectl rollout restart statefulset test-nifi-node-default --namespace $NAMESPACE diff --git a/tests/templates/kuttl/smoke/test_nifi.py b/tests/templates/kuttl/smoke/test_nifi.py index b885c09f..9d89fb63 100755 --- a/tests/templates/kuttl/smoke/test_nifi.py +++ b/tests/templates/kuttl/smoke/test_nifi.py @@ -42,7 +42,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args['user'], args['password']) headers = {'Authorization': token} node_count = int(args['count']) diff --git a/tests/templates/kuttl/smoke/test_nifi_metrics.py b/tests/templates/kuttl/smoke/test_nifi_metrics.py index 86076038..2b0682e1 100755 --- a/tests/templates/kuttl/smoke/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke/test_nifi_metrics.py @@ -23,7 +23,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://nifi-node-default-0.nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 0336c48a..ce9a4b53 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -13,16 +13,10 @@ dimensions: - 1.27.0 - name: nifi_new values: - - 2.2.0 - # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version - # as in the example below. - # - 2.2.0,oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev - name: nifi-latest values: - - 2.2.0 - # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version - # as in the example below. - # - 2.2.0,oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev - name: zookeeper values: - 3.9.2 From fd729c23e7f0b191dbc58dccb55f244158cd0617 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 28 Apr 2025 13:13:01 +0200 Subject: [PATCH 02/13] clean up smoke test part 2 --- tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 index 6d0940e6..337c37a9 100644 --- a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 @@ -57,12 +57,12 @@ spec: provider: static: userCredentialsSecret: - name: nifi-admin-credentials + name: nifi-users --- apiVersion: v1 kind: Secret metadata: - name: nifi-admin-credentials + name: nifi-users stringData: admin: > passwordWithSpecialCharacter\@<&>"' From 663f0f0278196d9322c5a1ea9066a079faa7057f Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 29 Apr 2025 11:43:07 +0200 Subject: [PATCH 03/13] Add working test :) --- tests/release.yaml | 4 + .../kuttl/iceberg/00-patch-ns.yaml.j2 | 9 + .../iceberg/01-create-s3-connection.yaml | 5 + .../kuttl/iceberg/01_s3-connection.yaml | 36 + .../templates/kuttl/iceberg/10-assert.yaml.j2 | 10 + ...tor-aggregator-discovery-configmap.yaml.j2 | 9 + tests/templates/kuttl/iceberg/20-assert.yaml | 12 + .../kuttl/iceberg/20-install-minio.yaml | 5 + tests/templates/kuttl/iceberg/21-assert.yaml | 12 + .../iceberg/21-install-hive-postgres.yaml | 12 + .../21_helm-bitnami-postgresql-values.yaml.j2 | 29 + tests/templates/kuttl/iceberg/30-assert.yaml | 12 + .../kuttl/iceberg/30-install-hive.yaml.j2 | 42 + tests/templates/kuttl/iceberg/31-assert.yaml | 6 + .../kuttl/iceberg/31-install-trino.yaml.j2 | 53 ++ tests/templates/kuttl/iceberg/32-assert.yaml | 7 + .../kuttl/iceberg/32-create-iceberg-table.j2 | 22 + tests/templates/kuttl/iceberg/40-assert.yaml | 12 + .../iceberg/40-install-zookeeper.yaml.j2 | 21 + tests/templates/kuttl/iceberg/50-assert.yaml | 12 + .../kuttl/iceberg/50-install-nifi.yaml.j2 | 74 ++ .../60-create-nifi-flow-configmap.yaml | 5 + .../templates/kuttl/iceberg/60_nifi-flow.json | 801 ++++++++++++++++++ tests/templates/kuttl/iceberg/61-assert.yaml | 7 + .../kuttl/iceberg/61-provision-nifi-flow.yaml | 108 +++ tests/templates/kuttl/iceberg/70-assert.yaml | 7 + .../iceberg/70-check-iceberg-table.yaml.j2 | 34 + tests/templates/kuttl/iceberg/README.md | 32 + tests/test-definition.yaml | 17 + 29 files changed, 1415 insertions(+) create mode 100644 tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/01-create-s3-connection.yaml create mode 100644 tests/templates/kuttl/iceberg/01_s3-connection.yaml create mode 100644 tests/templates/kuttl/iceberg/10-assert.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/20-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/20-install-minio.yaml create mode 100644 tests/templates/kuttl/iceberg/21-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml create mode 100644 tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/30-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/30-install-hive.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/31-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/31-install-trino.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/32-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/32-create-iceberg-table.j2 create mode 100644 tests/templates/kuttl/iceberg/40-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/50-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml create mode 100644 tests/templates/kuttl/iceberg/60_nifi-flow.json create mode 100644 tests/templates/kuttl/iceberg/61-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml create mode 100644 tests/templates/kuttl/iceberg/70-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/README.md diff --git a/tests/release.yaml b/tests/release.yaml index b6ecf959..29e93a70 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -14,5 +14,9 @@ releases: operatorVersion: 0.0.0-dev zookeeper: operatorVersion: 0.0.0-dev + hive: + operatorVersion: 0.0.0-dev + trino: + operatorVersion: 0.0.0-dev nifi: operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 b/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml b/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml new file mode 100644 index 00000000..26f46c12 --- /dev/null +++ b/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst '$NAMESPACE' < 01_s3-connection.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/01_s3-connection.yaml b/tests/templates/kuttl/iceberg/01_s3-connection.yaml new file mode 100644 index 00000000..56c30d36 --- /dev/null +++ b/tests/templates/kuttl/iceberg/01_s3-connection.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: s3.stackable.tech/v1alpha1 +kind: S3Connection +metadata: + name: minio +spec: + host: "minio.${NAMESPACE}.svc.cluster.local" + port: 9000 + accessStyle: Path + credentials: + secretClass: s3-credentials-class + tls: + verification: + server: + caCert: + secretClass: tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: s3-credentials-class +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + labels: + secrets.stackable.tech/class: s3-credentials-class +stringData: + accessKey: admin + secretKey: adminadmin diff --git a/tests/templates/kuttl/iceberg/10-assert.yaml.j2 b/tests/templates/kuttl/iceberg/10-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/iceberg/10-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/iceberg/20-assert.yaml b/tests/templates/kuttl/iceberg/20-assert.yaml new file mode 100644 index 00000000..477bdd02 --- /dev/null +++ b/tests/templates/kuttl/iceberg/20-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/20-install-minio.yaml b/tests/templates/kuttl/iceberg/20-install-minio.yaml new file mode 100644 index 00000000..bbef7238 --- /dev/null +++ b/tests/templates/kuttl/iceberg/20-install-minio.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml diff --git a/tests/templates/kuttl/iceberg/21-assert.yaml b/tests/templates/kuttl/iceberg/21-assert.yaml new file mode 100644 index 00000000..1ac12423 --- /dev/null +++ b/tests/templates/kuttl/iceberg/21-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml b/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml new file mode 100644 index 00000000..fa5698e0 --- /dev/null +++ b/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: >- + helm upgrade postgresql + --install + --version=12.5.6 + --namespace $NAMESPACE + -f 21_helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..2526c3df --- /dev/null +++ b/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,29 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + extendedConfiguration: | + password_encryption=md5 + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "512Mi" + cpu: "512m" + limits: + memory: "512Mi" + cpu: "1" + +auth: + username: hive + password: hive + database: hive diff --git a/tests/templates/kuttl/iceberg/30-assert.yaml b/tests/templates/kuttl/iceberg/30-assert.yaml new file mode 100644 index 00000000..b893d15b --- /dev/null +++ b/tests/templates/kuttl/iceberg/30-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/iceberg/30-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-hive.yaml.j2 new file mode 100644 index 00000000..db3c7acb --- /dev/null +++ b/tests/templates/kuttl/iceberg/30-install-hive.yaml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: hive.stackable.tech/v1alpha1 +kind: HiveCluster +metadata: + name: hive +spec: + image: +{% if test_scenario['values']['hive-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hive-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hive-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hive-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + database: + connString: jdbc:postgresql://postgresql:5432/hive + credentialsSecret: postgres-credentials + dbType: postgres + # hdfs: + # configMap: hdfs + s3: + reference: minio +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + metastore: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 2 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/tests/templates/kuttl/iceberg/31-assert.yaml b/tests/templates/kuttl/iceberg/31-assert.yaml new file mode 100644 index 00000000..b9f1bbe1 --- /dev/null +++ b/tests/templates/kuttl/iceberg/31-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s diff --git a/tests/templates/kuttl/iceberg/31-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/31-install-trino.yaml.j2 new file mode 100644 index 00000000..5d2f1929 --- /dev/null +++ b/tests/templates/kuttl/iceberg/31-install-trino.yaml.j2 @@ -0,0 +1,53 @@ +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: trino +spec: + image: +{% if test_scenario['values']['trino-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + workers: + config: + gracefulShutdownTimeout: 60s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + config: {} +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: iceberg + labels: + trino: trino +spec: + connector: + iceberg: + metastore: + configMap: hive + s3: + reference: minio + # hdfs: + # configMap: hdfs diff --git a/tests/templates/kuttl/iceberg/32-assert.yaml b/tests/templates/kuttl/iceberg/32-assert.yaml new file mode 100644 index 00000000..c1a8246d --- /dev/null +++ b/tests/templates/kuttl/iceberg/32-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-iceberg-table +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/32-create-iceberg-table.j2 b/tests/templates/kuttl/iceberg/32-create-iceberg-table.j2 new file mode 100644 index 00000000..753eb14c --- /dev/null +++ b/tests/templates/kuttl/iceberg/32-create-iceberg-table.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-iceberg-table +spec: + template: + spec: + containers: + - name: create-iceberg-table + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" + command: + - bash + - -euo + - pipefail + - -c + - | + cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin + CREATE SCHEMA IF NOT EXISTS iceberg.test WITH (location = 's3a://demo/lakehouse/test'); + CREATE TABLE IF NOT EXISTS iceberg.test.greetings (hello varchar); + EOF + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/40-assert.yaml b/tests/templates/kuttl/iceberg/40-assert.yaml new file mode 100644 index 00000000..49ba7437 --- /dev/null +++ b/tests/templates/kuttl/iceberg/40-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2 new file mode 100644 index 00000000..58440246 --- /dev/null +++ b/tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: zookeeper +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/50-assert.yaml b/tests/templates/kuttl/iceberg/50-assert.yaml new file mode 100644 index 00000000..35aae31d --- /dev/null +++ b/tests/templates/kuttl/iceberg/50-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nifi-node-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 new file mode 100644 index 00000000..a6f70e33 --- /dev/null +++ b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 @@ -0,0 +1,74 @@ +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: nifi +spec: + image: +{% if test_scenario['values']['nifi-iceberg'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi-iceberg'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi-iceberg'].split(',')[0] }}" +{% else %} + custom: null + productVersion: "{{ test_scenario['values']['nifi-iceberg'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + zookeeperConfigMapName: nifi-znode + listenerClass: external-unstable + authentication: + - authenticationClass: nifi-users + sensitiveProperties: + keySecret: nifi-sensitive-property-key +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + nifi.properties: + # Quicker startup, and we only have a single node + nifi.cluster.flow.election.max.wait.time: 5 secs + jvmArgumentOverrides: + add: + # Needed for NiFi to trust the minio cert + - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 + - -Djavax.net.ssl.trustStorePassword=secret + - -Djavax.net.ssl.trustStoreType=PKCS12 + roleGroups: + default: + replicas: 1 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-users +stringData: + admin: adminadmin +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-znode +spec: + clusterRef: + name: zookeeper diff --git a/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml new file mode 100644 index 00000000..fe92ec83 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: cat 60_nifi-flow.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json new file mode 100644 index 00000000..23c679f1 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60_nifi-flow.json @@ -0,0 +1,801 @@ +{ + "externalControllerServices": {}, + "flowContents": { + "comments": "", + "componentType": "PROCESS_GROUP", + "connections": [ + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "id": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "16507d9c-da5d-3d39-9aa3-7e0a71b5851a", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e", + "instanceIdentifier": "e43fe911-835c-3352-aa1a-845f31de21c5", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "11cf418f-4873-3913-b573-a2b07322b81d", + "instanceIdentifier": "974a02ce-97c5-3903-a32b-09837c297f50", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "857e02bd-f9b2-3dde-a564-01e8594b1cb2", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "5737572d-d766-3b40-8612-5854ddb29b39", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f", + "instanceIdentifier": "f31cd9f5-16a1-3b74-929b-997f1c7446e7", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + } + ], + "controllerServices": [ + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-aws-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService" + }, + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService" + } + ], + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "instanceIdentifier": "9aea49ad-5078-385b-9890-7eae61aad31d", + "name": "AWSCredentialsProviderControllerService", + "properties": { + "anonymous-credentials": "false", + "Assume Role ARN": null, + "Assume Role Session Name": null, + "assume-role-external-id": null, + "assume-role-proxy-configuration-service": null, + "assume-role-ssl-context-service": null, + "assume-role-sts-endpoint": null, + "assume-role-sts-region": "us-west-2", + "assume-role-sts-signer-override": "Default Signature", + "Credentials File": null, + "custom-signer-class-name": null, + "custom-signer-module-location": null, + "default-credentials": "false", + "profile-name": null, + "Session Time": "3600", + "Access Key": "admin", + "Secret Key": "adminadmin" + }, + "propertyDescriptors": { + "Access Key": { + "displayName": "Access Key ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "Access Key", + "sensitive": true + }, + "anonymous-credentials": { + "displayName": "Use Anonymous Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "anonymous-credentials", + "sensitive": false + }, + "Assume Role ARN": { + "displayName": "Assume Role ARN", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role ARN", + "sensitive": false + }, + "Assume Role Session Name": { + "displayName": "Assume Role Session Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role Session Name", + "sensitive": false + }, + "assume-role-external-id": { + "displayName": "Assume Role External ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-external-id", + "sensitive": false + }, + "assume-role-proxy-configuration-service": { + "displayName": "Assume Role Proxy Configuration Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-proxy-configuration-service", + "sensitive": false + }, + "assume-role-ssl-context-service": { + "displayName": "Assume Role SSL Context Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-ssl-context-service", + "sensitive": false + }, + "assume-role-sts-endpoint": { + "displayName": "Assume Role STS Endpoint Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-endpoint", + "sensitive": false + }, + "assume-role-sts-region": { + "displayName": "Assume Role STS Region", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-region", + "sensitive": false + }, + "assume-role-sts-signer-override": { + "displayName": "Assume Role STS Signer Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-signer-override", + "sensitive": false + }, + "Credentials File": { + "displayName": "Credentials File", + "dynamic": false, + "identifiesControllerService": false, + "name": "Credentials File", + "resourceDefinition": { + "cardinality": "SINGLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "custom-signer-class-name": { + "displayName": "Custom Signer Class Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-class-name", + "sensitive": false + }, + "custom-signer-module-location": { + "displayName": "Custom Signer Module Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-module-location", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "DIRECTORY", + "FILE" + ] + }, + "sensitive": false + }, + "default-credentials": { + "displayName": "Use Default Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "default-credentials", + "sensitive": false + }, + "profile-name": { + "displayName": "Profile Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "profile-name", + "sensitive": false + }, + "Secret Key": { + "displayName": "Secret Access Key", + "dynamic": false, + "identifiesControllerService": false, + "name": "Secret Key", + "sensitive": true + }, + "Session Time": { + "displayName": "Assume Role Session Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "Session Time", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-record-serialization-services-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.serialization.RecordReaderFactory" + } + ], + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "instanceIdentifier": "654dcad1-4064-3fd9-9090-66434221e547", + "name": "JsonTreeReader", + "properties": { + "Allow Comments": "false", + "Date Format": null, + "Max String Length": "20 MB", + "schema-access-strategy": "infer-schema", + "schema-application-strategy": "SELECTED_PART", + "schema-branch": null, + "schema-inference-cache": null, + "schema-name": "${schema.name}", + "schema-reference-reader": null, + "schema-registry": null, + "schema-text": "${avro.schema}", + "schema-version": null, + "starting-field-name": null, + "starting-field-strategy": "ROOT_NODE", + "Time Format": null, + "Timestamp Format": null + }, + "propertyDescriptors": { + "Allow Comments": { + "displayName": "Allow Comments", + "dynamic": false, + "identifiesControllerService": false, + "name": "Allow Comments", + "sensitive": false + }, + "Date Format": { + "displayName": "Date Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Date Format", + "sensitive": false + }, + "Max String Length": { + "displayName": "Max String Length", + "dynamic": false, + "identifiesControllerService": false, + "name": "Max String Length", + "sensitive": false + }, + "schema-access-strategy": { + "displayName": "Schema Access Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-access-strategy", + "sensitive": false + }, + "schema-application-strategy": { + "displayName": "Schema Application Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-application-strategy", + "sensitive": false + }, + "schema-branch": { + "displayName": "Schema Branch", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-branch", + "sensitive": false + }, + "schema-inference-cache": { + "displayName": "Schema Inference Cache", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-inference-cache", + "sensitive": false + }, + "schema-name": { + "displayName": "Schema Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-name", + "sensitive": false + }, + "schema-reference-reader": { + "displayName": "Schema Reference Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-reference-reader", + "sensitive": false + }, + "schema-registry": { + "displayName": "Schema Registry", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-registry", + "sensitive": false + }, + "schema-text": { + "displayName": "Schema Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-text", + "sensitive": false + }, + "schema-version": { + "displayName": "Schema Version", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-version", + "sensitive": false + }, + "starting-field-name": { + "displayName": "Starting Field Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-name", + "sensitive": false + }, + "starting-field-strategy": { + "displayName": "Starting Field Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-strategy", + "sensitive": false + }, + "Time Format": { + "displayName": "Time Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Time Format", + "sensitive": false + }, + "Timestamp Format": { + "displayName": "Timestamp Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Timestamp Format", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.json.JsonTreeReader" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-services-nar", + "group": "tech.stackable.nifi", + "version": "0.0.1" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-iceberg-services-api-nar", + "group": "tech.stackable.nifi", + "version": "0.0.1" + }, + "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService" + } + ], + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "instanceIdentifier": "1fba3438-db48-3c0e-b80b-ea07b20b8b0d", + "name": "IcebergHiveCatalogService", + "properties": { + "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hive-metastore-uri": "thrift://hive:9083", + "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-path-style-access": "true", + "warehouse-location": "s3a://demo/lakehouse" + }, + "propertyDescriptors": { + "AWS Credentials Provider service": { + "displayName": "AWS Credentials Provider Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "AWS Credentials Provider service", + "sensitive": false + }, + "hive-metastore-uri": { + "displayName": "Hive Metastore URI", + "dynamic": false, + "identifiesControllerService": false, + "name": "hive-metastore-uri", + "sensitive": false + }, + "s3-endpoint": { + "displayName": "S3 endpoint", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-endpoint", + "sensitive": false + }, + "s3-path-style-access": { + "displayName": "S3 path style access", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-path-style-access", + "sensitive": false + }, + "warehouse-location": { + "displayName": "Default Warehouse Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "warehouse-location", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "tech.stackable.nifi.services.iceberg.IcebergHiveCatalogService" + } + ], + "defaultBackPressureDataSizeThreshold": "1 GB", + "defaultBackPressureObjectThreshold": 10000, + "defaultFlowFileExpiration": "0 sec", + "executionEngine": "INHERITED", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "funnels": [ + { + "componentType": "FUNNEL", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "5737572d-d766-3b40-8612-5854ddb29b39", + "position": { + "x": -416.0, + "y": -176.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "16507d9c-da5d-3d39-9aa3-7e0a71b5851a", + "position": { + "x": 88.0, + "y": -416.0 + } + } + ], + "identifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "inputPorts": [], + "instanceIdentifier": "80ce4814-0196-1000-0000-00006f596034", + "labels": [], + "maxConcurrentTasks": 1, + "name": "Iceberg Test", + "outputPorts": [], + "position": { + "x": 100.0, + "y": 10.0 + }, + "processGroups": [], + "processors": [ + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.1" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg", + "penaltyDuration": "30 sec", + "position": { + "x": -568.0, + "y": -456.0 + }, + "properties": { + "catalog-namespace": "test", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "PARQUET", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "857e02bd-f9b2-3dde-a564-01e8594b1cb2", + "maxBackoffPeriod": "10 mins", + "name": "GenerateFlowFile", + "penaltyDuration": "30 sec", + "position": { + "x": -568.0, + "y": -704.0 + }, + "properties": { + "Batch Size": "1", + "character-set": "UTF-8", + "Data Format": "Text", + "File Size": "0B", + "generate-ff-custom-text": "{\"hello\": \"world from NiFi :)\"}", + "mime-type": null, + "Unique FlowFiles": "false" + }, + "propertyDescriptors": { + "Batch Size": { + "displayName": "Batch Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "Batch Size", + "sensitive": false + }, + "character-set": { + "displayName": "Character Set", + "dynamic": false, + "identifiesControllerService": false, + "name": "character-set", + "sensitive": false + }, + "Data Format": { + "displayName": "Data Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Data Format", + "sensitive": false + }, + "File Size": { + "displayName": "File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "File Size", + "sensitive": false + }, + "generate-ff-custom-text": { + "displayName": "Custom Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "generate-ff-custom-text", + "sensitive": false + }, + "mime-type": { + "displayName": "Mime Type", + "dynamic": false, + "identifiesControllerService": false, + "name": "mime-type", + "sensitive": false + }, + "Unique FlowFiles": { + "displayName": "Unique FlowFiles", + "dynamic": false, + "identifiesControllerService": false, + "name": "Unique FlowFiles", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "1 min", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "yieldDuration": "1 sec" + } + ], + "remoteProcessGroups": [], + "scheduledState": "ENABLED", + "statelessFlowTimeout": "1 min" + }, + "flowEncodingVersion": "1.0", + "latest": false, + "parameterContexts": {}, + "parameterProviders": {} +} \ No newline at end of file diff --git a/tests/templates/kuttl/iceberg/61-assert.yaml b/tests/templates/kuttl/iceberg/61-assert.yaml new file mode 100644 index 00000000..ced45eb4 --- /dev/null +++ b/tests/templates/kuttl/iceberg/61-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: provision-nifi-flow +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml new file mode 100644 index 00000000..374487ad --- /dev/null +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -0,0 +1,108 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: provision-nifi-flow +spec: + template: + spec: + containers: + - name: provision-nifi-flow + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - python -u /tmp/script/script.py + volumeMounts: + - name: script + mountPath: /tmp/script + - name: nifi-flow + mountPath: /tmp/nifi-flow + - name: nifi-users + mountPath: /nifi-users + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: script + configMap: + name: provision-nifi-flow-script + - name: nifi-flow + configMap: + name: nifi-flow + - name: nifi-users + secret: + secretName: nifi-users + restartPolicy: OnFailure +--- +# Taken from https://github.com/stackabletech/demos/blob/1744be00054eec2827b2d9ef0a90645843cb0075/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml#L52 +apiVersion: v1 +kind: ConfigMap +metadata: + name: provision-nifi-flow-script +data: + script.py: | + from nipyapi.canvas import get_root_pg_id, schedule_process_group, list_all_controllers, schedule_controller + from nipyapi.security import service_login + import nipyapi + import os + import requests + import urllib3 + + # As of 2022-08-29 we cant use "https://nifi:8443" here because

The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.

+ ENDPOINT = f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" + USERNAME = "admin" + PASSWORD = open("/nifi-users/admin").read() + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + nipyapi.config.nifi_config.host = f"{ENDPOINT}/nifi-api" + nipyapi.config.nifi_config.verify_ssl = False + + print(f"Logging in as {USERNAME}") + service_login(username=USERNAME, password=PASSWORD) + print("Logged in") + + filename = "/tmp/nifi-flow/nifi-flow.json" + + pg_id = get_root_pg_id() + + if not nipyapi.config.nifi_config.api_client: + nipyapi.config.nifi_config.api_client = ApiClient() + + header_params = {} + header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json']) + header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data']) + + nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST', + path_params={'pg_id': pg_id}, + header_params=header_params, + _return_http_data_only=True, + post_params=[ + ('id', pg_id), + ('groupName', 'Iceberg Test'), + ('positionX', 100), + ('positionY', 10), + ('clientId', nipyapi.nifi.FlowApi().generate_client_id()), + ], + files={ + 'file': filename + }, + auth_settings=['tokenAuth']) + + max_retries = 2 + for _ in range(max_retries): + controllers = list_all_controllers(pg_id) + for controller in controllers: + if controller.component.state != "ENABLED": + try: + schedule_controller(controller, scheduled=True) + print(f"Scheduled controller: {controller.component.name}") + except Exception as e: + print(f"Failed to schedule controller {controller.component.name}: {e}") + + schedule_process_group(pg_id, scheduled=True) diff --git a/tests/templates/kuttl/iceberg/70-assert.yaml b/tests/templates/kuttl/iceberg/70-assert.yaml new file mode 100644 index 00000000..af4fc942 --- /dev/null +++ b/tests/templates/kuttl/iceberg/70-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: check-iceberg-table +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 b/tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 new file mode 100644 index 00000000..adeac268 --- /dev/null +++ b/tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: check-iceberg-table +spec: + template: + spec: + containers: + - name: check-iceberg-table + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" + command: + - bash + - -euo + - pipefail + - -c + - | + COUNT=$(cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin + SELECT COUNT(*) FROM iceberg.test.greetings WHERE hello = 'world from NiFi :)'; + EOF + ) + + COUNT="${COUNT%\"}" # Remove trailing quote if any + COUNT="${COUNT#\"}" # Remove leading quote if any + echo "Count is $COUNT" + + # Check if it's a number greater than 0 + if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -le 0 ]; then + echo "Invalid or zero count: $COUNT" + exit 1 + fi + + echo "Count $COUNT was valid" + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md new file mode 100644 index 00000000..2fa467b7 --- /dev/null +++ b/tests/templates/kuttl/iceberg/README.md @@ -0,0 +1,32 @@ +The file `60_nifi-flow.json` was exported from the NiFi UI. + +*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO. + +Notable the following diff has been made: + +```diff +diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json +index 09783fa..23c679f 100644 +--- a/tests/templates/kuttl/iceberg/60_nifi-flow.json ++++ b/tests/templates/kuttl/iceberg/60_nifi-flow.json +@@ -160,7 +160,9 @@ + "custom-signer-module-location": null, + "default-credentials": "false", + "profile-name": null, +- "Session Time": "3600" ++ "Session Time": "3600", ++ "Access Key": "admin", ++ "Secret Key": "adminadmin" + }, + "propertyDescriptors": { + "Access Key": { +@@ -483,7 +485,7 @@ + "properties": { + "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hive-metastore-uri": "thrift://hive:9083", +- "s3-endpoint": "https://minio.kuttl-test-patient-tarpon.svc.cluster.local:9000", ++ "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-path-style-access": "true", + "warehouse-location": "s3a://demo/lakehouse" + }, +``` diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index ce9a4b53..c4ec9b5f 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -17,6 +17,10 @@ dimensions: - name: nifi-latest values: - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + - name: nifi-iceberg + values: + # - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + - 2.0.0,oci.stackable.tech/sandbox/sbernauer/nifi:2.2.0-stackable0.0.0-dev-with-iceberg - name: zookeeper values: - 3.9.2 @@ -24,6 +28,12 @@ dimensions: - name: zookeeper-latest values: - 3.9.3 + - name: hive-latest + values: + - 4.0.1 + - name: trino-latest + values: + - "470" - name: ldap-use-tls values: - "false" @@ -85,6 +95,13 @@ tests: - zookeeper-latest - oidc-use-tls - openshift + - name: iceberg + dimensions: + - nifi-iceberg + - zookeeper-latest + - hive-latest + - trino-latest + - openshift suites: - name: nightly patch: From 34d81c54a7f44191f4f5022a18427b614ed82ee2 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 29 Apr 2025 15:52:27 +0200 Subject: [PATCH 04/13] Move files --- tests/templates/kuttl/iceberg/30-assert.yaml | 12 ------------ tests/templates/kuttl/iceberg/31-assert.yaml | 12 +++++++++--- ...-install-hive.yaml.j2 => 31-install-hive.yaml.j2} | 0 tests/templates/kuttl/iceberg/32-assert.yaml | 11 +++++------ ...nstall-trino.yaml.j2 => 32-install-trino.yaml.j2} | 0 tests/templates/kuttl/iceberg/33-assert.yaml | 7 +++++++ ...e-iceberg-table.j2 => 33-create-iceberg-table.j2} | 0 7 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 tests/templates/kuttl/iceberg/30-assert.yaml rename tests/templates/kuttl/iceberg/{30-install-hive.yaml.j2 => 31-install-hive.yaml.j2} (100%) rename tests/templates/kuttl/iceberg/{31-install-trino.yaml.j2 => 32-install-trino.yaml.j2} (100%) create mode 100644 tests/templates/kuttl/iceberg/33-assert.yaml rename tests/templates/kuttl/iceberg/{32-create-iceberg-table.j2 => 33-create-iceberg-table.j2} (100%) diff --git a/tests/templates/kuttl/iceberg/30-assert.yaml b/tests/templates/kuttl/iceberg/30-assert.yaml deleted file mode 100644 index b893d15b..00000000 --- a/tests/templates/kuttl/iceberg/30-assert.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 900 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: hive-metastore-default -status: - readyReplicas: 2 - replicas: 2 diff --git a/tests/templates/kuttl/iceberg/31-assert.yaml b/tests/templates/kuttl/iceberg/31-assert.yaml index b9f1bbe1..b893d15b 100644 --- a/tests/templates/kuttl/iceberg/31-assert.yaml +++ b/tests/templates/kuttl/iceberg/31-assert.yaml @@ -1,6 +1,12 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 -commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/iceberg/30-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/31-install-hive.yaml.j2 similarity index 100% rename from tests/templates/kuttl/iceberg/30-install-hive.yaml.j2 rename to tests/templates/kuttl/iceberg/31-install-hive.yaml.j2 diff --git a/tests/templates/kuttl/iceberg/32-assert.yaml b/tests/templates/kuttl/iceberg/32-assert.yaml index c1a8246d..b9f1bbe1 100644 --- a/tests/templates/kuttl/iceberg/32-assert.yaml +++ b/tests/templates/kuttl/iceberg/32-assert.yaml @@ -1,7 +1,6 @@ --- -apiVersion: batch/v1 -kind: Job -metadata: - name: create-iceberg-table -status: - succeeded: 1 +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s diff --git a/tests/templates/kuttl/iceberg/31-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/32-install-trino.yaml.j2 similarity index 100% rename from tests/templates/kuttl/iceberg/31-install-trino.yaml.j2 rename to tests/templates/kuttl/iceberg/32-install-trino.yaml.j2 diff --git a/tests/templates/kuttl/iceberg/33-assert.yaml b/tests/templates/kuttl/iceberg/33-assert.yaml new file mode 100644 index 00000000..c1a8246d --- /dev/null +++ b/tests/templates/kuttl/iceberg/33-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-iceberg-table +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/32-create-iceberg-table.j2 b/tests/templates/kuttl/iceberg/33-create-iceberg-table.j2 similarity index 100% rename from tests/templates/kuttl/iceberg/32-create-iceberg-table.j2 rename to tests/templates/kuttl/iceberg/33-create-iceberg-table.j2 From dd4a1808033c97519f551435125d283607b7647d Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 2 May 2025 09:05:57 +0200 Subject: [PATCH 05/13] Add and test HDFS functionality --- tests/release.yaml | 2 + tests/templates/kuttl/iceberg/20-assert.yaml | 7 + tests/templates/kuttl/iceberg/30-assert.yaml | 12 + ...r.yaml.j2 => 30-install-zookeeper.yaml.j2} | 0 tests/templates/kuttl/iceberg/31-assert.yaml | 18 +- .../kuttl/iceberg/31-install-hdfs.j2 | 49 +++ tests/templates/kuttl/iceberg/32-assert.yaml | 12 +- ...l-hive.yaml.j2 => 32-install-hive.yaml.j2} | 4 +- tests/templates/kuttl/iceberg/33-assert.yaml | 11 +- ...trino.yaml.j2 => 33-install-trino.yaml.j2} | 5 +- tests/templates/kuttl/iceberg/40-assert.yaml | 13 +- ...g-table.j2 => 40-create-iceberg-tables.j2} | 10 +- .../kuttl/iceberg/50-install-nifi.yaml.j2 | 4 + .../templates/kuttl/iceberg/60_nifi-flow.json | 365 +++++++++++++++--- tests/templates/kuttl/iceberg/61-assert.yaml | 4 + .../kuttl/iceberg/61-provision-nifi-flow.yaml | 8 +- tests/templates/kuttl/iceberg/70-assert.yaml | 6 +- .../iceberg/70-check-iceberg-table.yaml.j2 | 34 -- .../iceberg/70-check-iceberg-tables.yaml.j2 | 37 ++ tests/templates/kuttl/iceberg/README.md | 2 + tests/test-definition.yaml | 4 + 21 files changed, 495 insertions(+), 112 deletions(-) create mode 100644 tests/templates/kuttl/iceberg/30-assert.yaml rename tests/templates/kuttl/iceberg/{40-install-zookeeper.yaml.j2 => 30-install-zookeeper.yaml.j2} (100%) create mode 100644 tests/templates/kuttl/iceberg/31-install-hdfs.j2 rename tests/templates/kuttl/iceberg/{31-install-hive.yaml.j2 => 32-install-hive.yaml.j2} (96%) rename tests/templates/kuttl/iceberg/{32-install-trino.yaml.j2 => 33-install-trino.yaml.j2} (94%) rename tests/templates/kuttl/iceberg/{33-create-iceberg-table.j2 => 40-create-iceberg-tables.j2} (56%) delete mode 100644 tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 diff --git a/tests/release.yaml b/tests/release.yaml index 29e93a70..ec44026c 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -14,6 +14,8 @@ releases: operatorVersion: 0.0.0-dev zookeeper: operatorVersion: 0.0.0-dev + hdfs: + operatorVersion: 0.0.0-dev hive: operatorVersion: 0.0.0-dev trino: diff --git a/tests/templates/kuttl/iceberg/20-assert.yaml b/tests/templates/kuttl/iceberg/20-assert.yaml index 477bdd02..e1829b77 100644 --- a/tests/templates/kuttl/iceberg/20-assert.yaml +++ b/tests/templates/kuttl/iceberg/20-assert.yaml @@ -10,3 +10,10 @@ metadata: status: readyReplicas: 1 replicas: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-post-job +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/30-assert.yaml b/tests/templates/kuttl/iceberg/30-assert.yaml new file mode 100644 index 00000000..49ba7437 --- /dev/null +++ b/tests/templates/kuttl/iceberg/30-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 similarity index 100% rename from tests/templates/kuttl/iceberg/40-install-zookeeper.yaml.j2 rename to tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 diff --git a/tests/templates/kuttl/iceberg/31-assert.yaml b/tests/templates/kuttl/iceberg/31-assert.yaml index b893d15b..126b5639 100644 --- a/tests/templates/kuttl/iceberg/31-assert.yaml +++ b/tests/templates/kuttl/iceberg/31-assert.yaml @@ -6,7 +6,23 @@ timeout: 900 apiVersion: apps/v1 kind: StatefulSet metadata: - name: hive-metastore-default + name: hdfs-namenode-default status: readyReplicas: 2 replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-journalnode-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-datanode-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/31-install-hdfs.j2 b/tests/templates/kuttl/iceberg/31-install-hdfs.j2 new file mode 100644 index 00000000..5687b79e --- /dev/null +++ b/tests/templates/kuttl/iceberg/31-install-hdfs.j2 @@ -0,0 +1,49 @@ +--- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: hdfs +spec: + image: +{% if test_scenario['values']['hdfs-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hdfs-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hdfs-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hdfs-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + dfsReplication: 1 +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + zookeeperConfigMapName: hdfs-znode + nameNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 2 + dataNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + journalNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: hdfs-znode +spec: + clusterRef: + name: zookeeper diff --git a/tests/templates/kuttl/iceberg/32-assert.yaml b/tests/templates/kuttl/iceberg/32-assert.yaml index b9f1bbe1..b893d15b 100644 --- a/tests/templates/kuttl/iceberg/32-assert.yaml +++ b/tests/templates/kuttl/iceberg/32-assert.yaml @@ -1,6 +1,12 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 -commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/iceberg/31-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/32-install-hive.yaml.j2 similarity index 96% rename from tests/templates/kuttl/iceberg/31-install-hive.yaml.j2 rename to tests/templates/kuttl/iceberg/32-install-hive.yaml.j2 index db3c7acb..d1a2a346 100644 --- a/tests/templates/kuttl/iceberg/31-install-hive.yaml.j2 +++ b/tests/templates/kuttl/iceberg/32-install-hive.yaml.j2 @@ -17,8 +17,8 @@ spec: connString: jdbc:postgresql://postgresql:5432/hive credentialsSecret: postgres-credentials dbType: postgres - # hdfs: - # configMap: hdfs + hdfs: + configMap: hdfs s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/iceberg/33-assert.yaml b/tests/templates/kuttl/iceberg/33-assert.yaml index c1a8246d..b9f1bbe1 100644 --- a/tests/templates/kuttl/iceberg/33-assert.yaml +++ b/tests/templates/kuttl/iceberg/33-assert.yaml @@ -1,7 +1,6 @@ --- -apiVersion: batch/v1 -kind: Job -metadata: - name: create-iceberg-table -status: - succeeded: 1 +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s diff --git a/tests/templates/kuttl/iceberg/32-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 similarity index 94% rename from tests/templates/kuttl/iceberg/32-install-trino.yaml.j2 rename to tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 index 5d2f1929..dbc3cacc 100644 --- a/tests/templates/kuttl/iceberg/32-install-trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 @@ -16,6 +16,7 @@ spec: catalogLabelSelector: matchLabels: trino: trino + listenerClass: external-unstable {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -49,5 +50,5 @@ spec: configMap: hive s3: reference: minio - # hdfs: - # configMap: hdfs + hdfs: + configMap: hdfs diff --git a/tests/templates/kuttl/iceberg/40-assert.yaml b/tests/templates/kuttl/iceberg/40-assert.yaml index 49ba7437..485373c8 100644 --- a/tests/templates/kuttl/iceberg/40-assert.yaml +++ b/tests/templates/kuttl/iceberg/40-assert.yaml @@ -1,12 +1,7 @@ --- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: batch/v1 +kind: Job metadata: - name: zookeeper-server-default + name: create-iceberg-tables status: - readyReplicas: 1 - replicas: 1 + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/33-create-iceberg-table.j2 b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 similarity index 56% rename from tests/templates/kuttl/iceberg/33-create-iceberg-table.j2 rename to tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 index 753eb14c..2105ac96 100644 --- a/tests/templates/kuttl/iceberg/33-create-iceberg-table.j2 +++ b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 @@ -2,12 +2,12 @@ apiVersion: batch/v1 kind: Job metadata: - name: create-iceberg-table + name: create-iceberg-tables spec: template: spec: containers: - - name: create-iceberg-table + - name: create-iceberg-tables image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" command: - bash @@ -16,7 +16,9 @@ spec: - -c - | cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin - CREATE SCHEMA IF NOT EXISTS iceberg.test WITH (location = 's3a://demo/lakehouse/test'); - CREATE TABLE IF NOT EXISTS iceberg.test.greetings (hello varchar); + CREATE SCHEMA IF NOT EXISTS iceberg.s3 WITH (location = 's3a://demo/lakehouse/s3'); + CREATE TABLE IF NOT EXISTS iceberg.s3.greetings (hello varchar); + CREATE SCHEMA IF NOT EXISTS iceberg.hdfs WITH (location = 'hdfs:/lakehouse/hdfs'); + CREATE TABLE IF NOT EXISTS iceberg.hdfs.greetings (hello varchar); EOF restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 index a6f70e33..99e8bd22 100644 --- a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 @@ -20,6 +20,10 @@ spec: - authenticationClass: nifi-users sensitiveProperties: keySecret: nifi-sensitive-property-key + extraVolumes: + - name: hdfs-config + configMap: + name: hdfs {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json index 23c679f1..2c6ed7d1 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow.json @@ -11,16 +11,16 @@ "componentType": "CONNECTION", "destination": { "comments": "", - "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", "id": "0ccd3baf-e0f5-3935-b660-456082596e69", - "instanceIdentifier": "16507d9c-da5d-3d39-9aa3-7e0a71b5851a", + "instanceIdentifier": "9013ea63-2dbe-3774-83bf-97110f8adf27", "name": "Funnel", "type": "FUNNEL" }, "flowFileExpiration": "0 sec", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e", - "instanceIdentifier": "e43fe911-835c-3352-aa1a-845f31de21c5", + "instanceIdentifier": "1b7d4c80-a27c-32e5-9428-0fce2f8a00bd", "labelIndex": 0, "loadBalanceCompression": "DO_NOT_COMPRESS", "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", @@ -32,9 +32,9 @@ ], "source": { "comments": "", - "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", - "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", "name": "PutIceberg", "type": "PROCESSOR" }, @@ -47,16 +47,124 @@ "componentType": "CONNECTION", "destination": { "comments": "", - "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "8fbbad6f-0196-1000-0000-0000468ba4b3", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "73b06526-5744-3aad-b148-995a910f28c5", + "instanceIdentifier": "8fbbca9c-0196-1000-0000-00002d9b3b50", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "8fbcd068-0196-1000-0000-00003242b691", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "79b3e396-3be1-3175-8c52-4f93347b1f9d", + "instanceIdentifier": "8fbcdd98-0196-1000-0000-00003cc6f1be", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "bc79d195-d4b2-3531-846e-77318e9f08c0", + "instanceIdentifier": "8fbb639e-0196-1000-ffff-ffff9b19bf06", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", - "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", "name": "PutIceberg", "type": "PROCESSOR" }, "flowFileExpiration": "0 sec", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "11cf418f-4873-3913-b573-a2b07322b81d", - "instanceIdentifier": "974a02ce-97c5-3903-a32b-09837c297f50", + "instanceIdentifier": "c6e38b01-4f7f-34dd-8be1-22aec59413b6", "labelIndex": 0, "loadBalanceCompression": "DO_NOT_COMPRESS", "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", @@ -68,9 +176,9 @@ ], "source": { "comments": "", - "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", "id": "27782850-84c8-37e6-a437-a74b2308e57e", - "instanceIdentifier": "857e02bd-f9b2-3dde-a564-01e8594b1cb2", + "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653", "name": "GenerateFlowFile", "type": "PROCESSOR" }, @@ -83,16 +191,16 @@ "componentType": "CONNECTION", "destination": { "comments": "", - "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4", - "instanceIdentifier": "5737572d-d766-3b40-8612-5854ddb29b39", + "instanceIdentifier": "6d38af9a-d0e9-3b2e-a776-44da91f4497d", "name": "Funnel", "type": "FUNNEL" }, "flowFileExpiration": "0 sec", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f", - "instanceIdentifier": "f31cd9f5-16a1-3b74-929b-997f1c7446e7", + "instanceIdentifier": "195765f2-2d5d-3221-9e72-fb39091ad4d0", "labelIndex": 0, "loadBalanceCompression": "DO_NOT_COMPRESS", "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", @@ -104,9 +212,9 @@ ], "source": { "comments": "", - "groupId": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", - "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", "name": "PutIceberg", "type": "PROCESSOR" }, @@ -130,7 +238,7 @@ "group": "org.apache.nifi", "version": "2.2.0" }, - "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService" + "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService" }, { "bundle": { @@ -138,12 +246,12 @@ "group": "org.apache.nifi", "version": "2.2.0" }, - "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService" + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService" } ], - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7", - "instanceIdentifier": "9aea49ad-5078-385b-9890-7eae61aad31d", + "instanceIdentifier": "a4049610-9d6c-3a38-a411-06fe8e5b7425", "name": "AWSCredentialsProviderControllerService", "properties": { "anonymous-credentials": "false", @@ -263,8 +371,8 @@ "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ - "DIRECTORY", - "FILE" + "FILE", + "DIRECTORY" ] }, "sensitive": false @@ -320,9 +428,9 @@ "type": "org.apache.nifi.serialization.RecordReaderFactory" } ], - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe", - "instanceIdentifier": "654dcad1-4064-3fd9-9090-66434221e547", + "instanceIdentifier": "15771472-fd85-37b8-9b4d-f496e31f49c2", "name": "JsonTreeReader", "properties": { "Allow Comments": "false", @@ -464,7 +572,7 @@ "bundle": { "artifact": "nifi-iceberg-services-nar", "group": "tech.stackable.nifi", - "version": "0.0.1" + "version": "0.0.2" }, "comments": "", "componentType": "CONTROLLER_SERVICE", @@ -473,17 +581,18 @@ "bundle": { "artifact": "nifi-iceberg-services-api-nar", "group": "tech.stackable.nifi", - "version": "0.0.1" + "version": "0.0.2" }, "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService" } ], - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4", - "instanceIdentifier": "1fba3438-db48-3c0e-b80b-ea07b20b8b0d", + "instanceIdentifier": "127f71a3-94c7-3c71-b761-36f2a09629a7", "name": "IcebergHiveCatalogService", "properties": { "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml", "hive-metastore-uri": "thrift://hive:9083", "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", @@ -497,6 +606,19 @@ "name": "AWS Credentials Provider service", "sensitive": false }, + "hadoop-config-resources": { + "displayName": "Hadoop Configuration Resources", + "dynamic": false, + "identifiesControllerService": false, + "name": "hadoop-config-resources", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, "hive-metastore-uri": { "displayName": "Hive Metastore URI", "dynamic": false, @@ -539,28 +661,48 @@ "funnels": [ { "componentType": "FUNNEL", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "8fbcd068-0196-1000-0000-00003242b691", + "position": { + "x": -880.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "8fbbad6f-0196-1000-0000-0000468ba4b3", + "position": { + "x": -1344.0, + "y": -432.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4", - "instanceIdentifier": "5737572d-d766-3b40-8612-5854ddb29b39", + "instanceIdentifier": "6d38af9a-d0e9-3b2e-a776-44da91f4497d", "position": { - "x": -416.0, - "y": -176.0 + "x": -328.0, + "y": -200.0 } }, { "componentType": "FUNNEL", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69", - "instanceIdentifier": "16507d9c-da5d-3d39-9aa3-7e0a71b5851a", + "instanceIdentifier": "9013ea63-2dbe-3774-83bf-97110f8adf27", "position": { - "x": 88.0, - "y": -416.0 + "x": 160.0, + "y": -432.0 } } ], - "identifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "identifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "inputPorts": [], - "instanceIdentifier": "80ce4814-0196-1000-0000-00006f596034", + "instanceIdentifier": "8fb66dc6-0196-1000-ffff-ffff98cfc281", "labels": [], "maxConcurrentTasks": 1, "name": "Iceberg Test", @@ -578,24 +720,149 @@ "bundle": { "artifact": "nifi-iceberg-processors-nar", "group": "tech.stackable.nifi", - "version": "0.0.1" + "version": "0.0.2" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg", + "penaltyDuration": "30 sec", + "position": { + "x": -1032.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "hdfs", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "ORC", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.2" }, "comments": "", "componentType": "PROCESSOR", "concurrentlySchedulableTaskCount": 1, "executionNode": "ALL", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b", - "instanceIdentifier": "ae5bb10d-a92c-3066-883a-beb2a239441b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", "maxBackoffPeriod": "10 mins", "name": "PutIceberg", "penaltyDuration": "30 sec", "position": { - "x": -568.0, - "y": -456.0 + "x": -480.0, + "y": -472.0 }, "properties": { - "catalog-namespace": "test", + "catalog-namespace": "s3", "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", "file-format": "PARQUET", "maximum-commit-duration": "30 sec", @@ -709,14 +976,14 @@ "componentType": "PROCESSOR", "concurrentlySchedulableTaskCount": 1, "executionNode": "ALL", - "groupIdentifier": "3898fb3b-0ba0-3e28-ad83-27d65179d012", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", "identifier": "27782850-84c8-37e6-a437-a74b2308e57e", - "instanceIdentifier": "857e02bd-f9b2-3dde-a564-01e8594b1cb2", + "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653", "maxBackoffPeriod": "10 mins", "name": "GenerateFlowFile", "penaltyDuration": "30 sec", "position": { - "x": -568.0, + "x": -744.0, "y": -704.0 }, "properties": { diff --git a/tests/templates/kuttl/iceberg/61-assert.yaml b/tests/templates/kuttl/iceberg/61-assert.yaml index ced45eb4..813504b1 100644 --- a/tests/templates/kuttl/iceberg/61-assert.yaml +++ b/tests/templates/kuttl/iceberg/61-assert.yaml @@ -1,4 +1,8 @@ --- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- apiVersion: batch/v1 kind: Job metadata: diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml index 374487ad..09cd08fa 100644 --- a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -70,6 +70,7 @@ data: filename = "/tmp/nifi-flow/nifi-flow.json" pg_id = get_root_pg_id() + print(f"Got root process group id: {pg_id}") if not nipyapi.config.nifi_config.api_client: nipyapi.config.nifi_config.api_client = ApiClient() @@ -78,6 +79,7 @@ data: header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json']) header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data']) + print("Uploading process group") nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST', path_params={'pg_id': pg_id}, header_params=header_params, @@ -93,13 +95,17 @@ data: 'file': filename }, auth_settings=['tokenAuth']) + print("Process group uploaded") - max_retries = 2 + # As they are started in the wrong order :D we need to retry + max_retries = 3 for _ in range(max_retries): controllers = list_all_controllers(pg_id) + print(f"Found {len(controllers)} controllers") for controller in controllers: if controller.component.state != "ENABLED": try: + print(f"Scheduling controller {controller.component.name}") schedule_controller(controller, scheduled=True) print(f"Scheduled controller: {controller.component.name}") except Exception as e: diff --git a/tests/templates/kuttl/iceberg/70-assert.yaml b/tests/templates/kuttl/iceberg/70-assert.yaml index af4fc942..4a7bd272 100644 --- a/tests/templates/kuttl/iceberg/70-assert.yaml +++ b/tests/templates/kuttl/iceberg/70-assert.yaml @@ -1,7 +1,11 @@ --- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- apiVersion: batch/v1 kind: Job metadata: - name: check-iceberg-table + name: check-iceberg-tables status: succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 b/tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 deleted file mode 100644 index adeac268..00000000 --- a/tests/templates/kuttl/iceberg/70-check-iceberg-table.yaml.j2 +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: check-iceberg-table -spec: - template: - spec: - containers: - - name: check-iceberg-table - image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" - command: - - bash - - -euo - - pipefail - - -c - - | - COUNT=$(cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin - SELECT COUNT(*) FROM iceberg.test.greetings WHERE hello = 'world from NiFi :)'; - EOF - ) - - COUNT="${COUNT%\"}" # Remove trailing quote if any - COUNT="${COUNT#\"}" # Remove leading quote if any - echo "Count is $COUNT" - - # Check if it's a number greater than 0 - if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -le 0 ]; then - echo "Invalid or zero count: $COUNT" - exit 1 - fi - - echo "Count $COUNT was valid" - restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 new file mode 100644 index 00000000..184675aa --- /dev/null +++ b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 @@ -0,0 +1,37 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: check-iceberg-tables +spec: + template: + spec: + containers: + - name: check-iceberg-tables + # image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" + image: "oci.stackable.tech/sdp/trino-cli:470-stackable0.0.0-dev" + command: + - bash + - -euo + - pipefail + - -c + - | + for SCHEMA in iceberg.s3 iceberg.hdfs; do + COUNT=$(cat << EOF | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin + SELECT COUNT(*) FROM $SCHEMA.greetings WHERE hello = 'world from NiFi :)'; + EOF + ) + + COUNT="${COUNT%\"}" # Remove trailing quote if any + COUNT="${COUNT#\"}" # Remove leading quote if any + echo "Count is $COUNT" + + # Check if it's a number greater than 0 + if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -le 0 ]; then + echo "Invalid or zero count: $COUNT" + exit 1 + fi + + echo "Count $COUNT was valid" + done + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md index 2fa467b7..da3e9d1d 100644 --- a/tests/templates/kuttl/iceberg/README.md +++ b/tests/templates/kuttl/iceberg/README.md @@ -2,6 +2,8 @@ The file `60_nifi-flow.json` was exported from the NiFi UI. *However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO. +TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier. + Notable the following diff has been made: ```diff diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index c4ec9b5f..6b3e75c2 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -28,6 +28,9 @@ dimensions: - name: zookeeper-latest values: - 3.9.3 + - name: hdfs-latest + values: + - 3.4.0 - name: hive-latest values: - 4.0.1 @@ -99,6 +102,7 @@ tests: dimensions: - nifi-iceberg - zookeeper-latest + - hdfs-latest - hive-latest - trino-latest - openshift From bf5691588668b903cb2857d4feb421ca9c7154bc Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 2 May 2025 15:15:22 +0200 Subject: [PATCH 06/13] Kerbize HDFS and HMS --- tests/release.yaml | 2 + tests/templates/kuttl/iceberg/00-rbac.yaml.j2 | 29 ++++ .../templates/kuttl/iceberg/02-assert.yaml.j2 | 14 ++ .../kuttl/iceberg/02-install-krb5-kdc.yaml.j2 | 146 ++++++++++++++++++ .../03-create-kerberos-secretclass.yaml.j2 | 8 + .../iceberg/03_kerberos-secretclass.yaml.j2 | 33 ++++ tests/templates/kuttl/iceberg/31-assert.yaml | 28 ---- .../kuttl/iceberg/31-install-hdfs.j2 | 49 ------ tests/templates/kuttl/iceberg/31-opa.yaml.j2 | 17 ++ tests/templates/kuttl/iceberg/32-assert.yaml | 18 ++- .../kuttl/iceberg/32-install-hdfs.yaml.j2 | 6 + tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 | 78 ++++++++++ tests/templates/kuttl/iceberg/33-assert.yaml | 12 +- .../kuttl/iceberg/33-install-hive.yaml.j2 | 6 + .../kuttl/iceberg/33-install-trino.yaml.j2 | 54 ------- ...2-install-hive.yaml.j2 => 33_hive.yaml.j2} | 15 +- tests/templates/kuttl/iceberg/34-assert.yaml | 6 + .../kuttl/iceberg/34-install-trino.yaml.j2 | 6 + .../templates/kuttl/iceberg/34_trino.yaml.j2 | 108 +++++++++++++ .../kuttl/iceberg/40-create-iceberg-tables.j2 | 2 +- .../kuttl/iceberg/50-install-nifi.yaml.j2 | 82 +--------- tests/templates/kuttl/iceberg/50_nifi.yaml.j2 | 111 +++++++++++++ .../iceberg/70-check-iceberg-tables.yaml.j2 | 3 +- .../kuttl/iceberg/interactive-nifi.yaml | 47 ++++++ tests/test-definition.yaml | 33 +++- 25 files changed, 686 insertions(+), 227 deletions(-) create mode 100644 tests/templates/kuttl/iceberg/00-rbac.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/02-assert.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 delete mode 100644 tests/templates/kuttl/iceberg/31-assert.yaml delete mode 100644 tests/templates/kuttl/iceberg/31-install-hdfs.j2 create mode 100644 tests/templates/kuttl/iceberg/31-opa.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 delete mode 100644 tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 rename tests/templates/kuttl/iceberg/{32-install-hive.yaml.j2 => 33_hive.yaml.j2} (63%) create mode 100644 tests/templates/kuttl/iceberg/34-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/34_trino.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/50_nifi.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/interactive-nifi.yaml diff --git a/tests/release.yaml b/tests/release.yaml index ec44026c..04ee6498 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -12,6 +12,8 @@ releases: operatorVersion: 0.0.0-dev listener: operatorVersion: 0.0.0-dev + opa: + operatorVersion: 0.0.0-dev zookeeper: operatorVersion: 0.0.0-dev hdfs: diff --git a/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 b/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/iceberg/02-assert.yaml.j2 b/tests/templates/kuttl/iceberg/02-assert.yaml.j2 new file mode 100644 index 00000000..9d648d0b --- /dev/null +++ b/tests/templates/kuttl/iceberg/02-assert.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +status: + readyReplicas: 1 + replicas: 1 +{% endif %} diff --git a/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 b/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 new file mode 100644 index 00000000..15b1e04c --- /dev/null +++ b/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 @@ -0,0 +1,146 @@ +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +spec: + selector: + matchLabels: + app: krb5-kdc + template: + metadata: + labels: + app: krb5-kdc + spec: + serviceAccountName: test-sa + initContainers: + - name: init + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - sh + - -euo + - pipefail + - -c + - | + test -e /var/kerberos/krb5kdc/principal || kdb5_util create -s -P asdf + kadmin.local get_principal -terse root/admin || kadmin.local add_principal -pw asdf root/admin + # stackable-secret-operator principal must match the keytab specified in the SecretClass + kadmin.local get_principal -terse stackable-secret-operator || kadmin.local add_principal -e aes256-cts-hmac-sha384-192:normal -pw asdf stackable-secret-operator + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + containers: + - name: kdc + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - krb5kdc + - -n + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data +# Root permissions required on Openshift to access internal ports +{% if test_scenario['values']['openshift'] == "true" %} + securityContext: + runAsUser: 0 +{% endif %} + - name: kadmind + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - kadmind + - -nofork + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data +# Root permissions required on Openshift to access internal ports +{% if test_scenario['values']['openshift'] == "true" %} + securityContext: + runAsUser: 0 +{% endif %} + - name: client + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + tty: true + stdin: true + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + volumes: + - name: config + configMap: + name: krb5-kdc + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: krb5-kdc +spec: + selector: + app: krb5-kdc + ports: + - name: kadmin + port: 749 + - name: kdc + port: 88 + - name: kdc-udp + port: 88 + protocol: UDP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: krb5-kdc +data: + krb5.conf: | + [logging] + default = STDERR + kdc = STDERR + admin_server = STDERR + # default = FILE:/var/log/krb5libs.log + # kdc = FILE:/var/log/krb5kdc.log + # admin_server = FILE:/vaggr/log/kadmind.log + [libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = {{ test_scenario['values']['kerberos-realm'] }} + spake_preauth_groups = edwards25519 + [realms] + {{ test_scenario['values']['kerberos-realm'] }} = { + acl_file = /stackable/config/kadm5.acl + disable_encrypted_timestamp = false + } + [domain_realm] + .cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + kadm5.acl: | + root/admin *e + stackable-secret-operator *e +{% endif %} diff --git a/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..d3061645 --- /dev/null +++ b/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 @@ -0,0 +1,8 @@ +--- +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 03_kerberos-secretclass.yaml | kubectl apply -n $NAMESPACE -f - +{% endif %} diff --git a/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..e5033462 --- /dev/null +++ b/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 @@ -0,0 +1,33 @@ +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: kerberos-$NAMESPACE +spec: + backend: + kerberosKeytab: + realmName: {{ test_scenario['values']['kerberos-realm'] }} + kdc: krb5-kdc.$NAMESPACE.svc.cluster.local + admin: + mit: + kadminServer: krb5-kdc.$NAMESPACE.svc.cluster.local + adminKeytabSecret: + namespace: $NAMESPACE + name: secret-operator-keytab + adminPrincipal: stackable-secret-operator +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-operator-keytab +data: + # To create keytab. When promted enter password asdf + # cat | ktutil << 'EOF' + # list + # add_entry -password -p stackable-secret-operator@CLUSTER.LOCAL -k 1 -e aes256-cts-hmac-sha384-192 + # wkt /tmp/keytab + # EOF +{% if test_scenario['values']['kerberos-realm'] == 'CLUSTER.LOCAL' %} + keytab: BQIAAABdAAEADUNMVVNURVIuTE9DQUwAGXN0YWNrYWJsZS1zZWNyZXQtb3BlcmF0b3IAAAABZAYWIgEAFAAgm8MCZ8B//XF1tH92GciD6/usWUNAmBTZnZQxLua2TkgAAAAB +{% elif test_scenario['values']['kerberos-realm'] == 'PROD.MYCORP' %} + keytab: BQIAAABbAAEAC1BST0QuTVlDT1JQABlzdGFja2FibGUtc2VjcmV0LW9wZXJhdG9yAAAAAWQZa0EBABQAIC/EnFNejq/K5lX6tX+B3/tkI13TCzkPB7d2ggCIEzE8AAAAAQ== +{% endif %} diff --git a/tests/templates/kuttl/iceberg/31-assert.yaml b/tests/templates/kuttl/iceberg/31-assert.yaml deleted file mode 100644 index 126b5639..00000000 --- a/tests/templates/kuttl/iceberg/31-assert.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 900 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: hdfs-namenode-default -status: - readyReplicas: 2 - replicas: 2 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: hdfs-journalnode-default -status: - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: hdfs-datanode-default -status: - readyReplicas: 1 - replicas: 1 diff --git a/tests/templates/kuttl/iceberg/31-install-hdfs.j2 b/tests/templates/kuttl/iceberg/31-install-hdfs.j2 deleted file mode 100644 index 5687b79e..00000000 --- a/tests/templates/kuttl/iceberg/31-install-hdfs.j2 +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: hdfs.stackable.tech/v1alpha1 -kind: HdfsCluster -metadata: - name: hdfs -spec: - image: -{% if test_scenario['values']['hdfs-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['hdfs-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['hdfs-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['hdfs-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent - clusterConfig: - dfsReplication: 1 -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - zookeeperConfigMapName: hdfs-znode - nameNodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 2 - dataNodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 1 - journalNodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 1 ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperZnode -metadata: - name: hdfs-znode -spec: - clusterRef: - name: zookeeper diff --git a/tests/templates/kuttl/iceberg/31-opa.yaml.j2 b/tests/templates/kuttl/iceberg/31-opa.yaml.j2 new file mode 100644 index 00000000..179bfbc2 --- /dev/null +++ b/tests/templates/kuttl/iceberg/31-opa.yaml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: opa.stackable.tech/v1alpha1 +kind: OpaCluster +metadata: + name: opa +spec: + image: +{% if test_scenario['values']['opa-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opa-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opa-l'] }}" +{% endif %} + servers: + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/32-assert.yaml b/tests/templates/kuttl/iceberg/32-assert.yaml index b893d15b..126b5639 100644 --- a/tests/templates/kuttl/iceberg/32-assert.yaml +++ b/tests/templates/kuttl/iceberg/32-assert.yaml @@ -6,7 +6,23 @@ timeout: 900 apiVersion: apps/v1 kind: StatefulSet metadata: - name: hive-metastore-default + name: hdfs-namenode-default status: readyReplicas: 2 replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-journalnode-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-datanode-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 b/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 new file mode 100644 index 00000000..9bbe6c50 --- /dev/null +++ b/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 32_hdfs.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 b/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 new file mode 100644 index 00000000..7a23c747 --- /dev/null +++ b/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 @@ -0,0 +1,78 @@ +--- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: hdfs +spec: + image: +{% if test_scenario['values']['hdfs-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hdfs-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hdfs-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hdfs-l'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + dfsReplication: 1 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + authentication: + tlsSecretClass: tls + kerberos: + secretClass: kerberos-$NAMESPACE + authorization: + opa: + configMapName: opa + package: hdfs +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + zookeeperConfigMapName: hdfs-znode + nameNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: &configOverrides + core-site.xml: + # The idea is that the user "hive" can't do anything in hdfs, + # *but* it can impersonate other users (such as trino), + # that have the needed permissions + hadoop.proxyuser.hive.users: "*" + hadoop.proxyuser.hive.hosts: "*" + roleGroups: + default: + replicas: 2 + dataNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + journalNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: hdfs-znode +spec: + clusterRef: + name: zookeeper +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hdfs-regorules + labels: + opa.stackable.tech/bundle: "true" +data: + hdfs.rego: | + package hdfs + + default allow := true diff --git a/tests/templates/kuttl/iceberg/33-assert.yaml b/tests/templates/kuttl/iceberg/33-assert.yaml index b9f1bbe1..50c27fd9 100644 --- a/tests/templates/kuttl/iceberg/33-assert.yaml +++ b/tests/templates/kuttl/iceberg/33-assert.yaml @@ -1,6 +1,12 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 -commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 new file mode 100644 index 00000000..126fe2ab --- /dev/null +++ b/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 33_hive.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 deleted file mode 100644 index dbc3cacc..00000000 --- a/tests/templates/kuttl/iceberg/33-install-trino.yaml.j2 +++ /dev/null @@ -1,54 +0,0 @@ ---- -apiVersion: trino.stackable.tech/v1alpha1 -kind: TrinoCluster -metadata: - name: trino -spec: - image: -{% if test_scenario['values']['trino-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['trino-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['trino-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['trino-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent - clusterConfig: - catalogLabelSelector: - matchLabels: - trino: trino - listenerClass: external-unstable -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - coordinators: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 1 - workers: - config: - gracefulShutdownTimeout: 60s # Let the test run faster - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - replicas: 1 - config: {} ---- -apiVersion: trino.stackable.tech/v1alpha1 -kind: TrinoCatalog -metadata: - name: iceberg - labels: - trino: trino -spec: - connector: - iceberg: - metastore: - configMap: hive - s3: - reference: minio - hdfs: - configMap: hdfs diff --git a/tests/templates/kuttl/iceberg/32-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/33_hive.yaml.j2 similarity index 63% rename from tests/templates/kuttl/iceberg/32-install-hive.yaml.j2 rename to tests/templates/kuttl/iceberg/33_hive.yaml.j2 index d1a2a346..ad533522 100644 --- a/tests/templates/kuttl/iceberg/32-install-hive.yaml.j2 +++ b/tests/templates/kuttl/iceberg/33_hive.yaml.j2 @@ -5,11 +5,11 @@ metadata: name: hive spec: image: -{% if test_scenario['values']['hive-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['hive-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['hive-latest'].split(',')[0] }}" +{% if test_scenario['values']['hive-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hive-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hive-l'].split(',')[0] }}" {% else %} - productVersion: "{{ test_scenario['values']['hive-latest'] }}" + productVersion: "{{ test_scenario['values']['hive-l'] }}" {% endif %} pullPolicy: IfNotPresent clusterConfig: @@ -21,6 +21,11 @@ spec: configMap: hdfs s3: reference: minio +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + authentication: + kerberos: + secretClass: kerberos-$NAMESPACE +{% endif %} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -30,7 +35,7 @@ spec: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: - replicas: 2 + replicas: 1 --- apiVersion: v1 kind: Secret diff --git a/tests/templates/kuttl/iceberg/34-assert.yaml b/tests/templates/kuttl/iceberg/34-assert.yaml new file mode 100644 index 00000000..b9f1bbe1 --- /dev/null +++ b/tests/templates/kuttl/iceberg/34-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s diff --git a/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 new file mode 100644 index 00000000..3bfdc0d2 --- /dev/null +++ b/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 34_trino.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 new file mode 100644 index 00000000..86491360 --- /dev/null +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -0,0 +1,108 @@ +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: trino +spec: + image: +{% if test_scenario['values']['trino-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino-l'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino + listenerClass: external-unstable +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: &podOverrides + spec: + containers: + - name: trino + env: + - name: KERBEROS_REALM + value: {{ test_scenario['values']['kerberos-realm'] }} + volumeMounts: + - name: kerberos + mountPath: /stackable/kerberos + # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`, + # but we can not influence the JVM arguments (edit: we can now). + - name: kerberos + mountPath: /etc/krb5.conf + subPath: krb5.conf + volumes: + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: trino + secrets.stackable.tech/scope: service=trino + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} + roleGroups: + default: + replicas: 1 + workers: + config: + gracefulShutdownTimeout: 60s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: *podOverrides +{% endif %} + roleGroups: + default: + replicas: 1 + config: {} +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: iceberg + labels: + trino: trino +spec: + connector: + iceberg: + metastore: + configMap: hive + s3: + reference: minio + hdfs: + configMap: hdfs +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + configOverrides: + # HDFS configuration + hive.hdfs.authentication.type: KERBEROS + hive.hdfs.trino.principal: trino/trino.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + hive.hdfs.trino.keytab: /stackable/kerberos/keytab + hive.hdfs.impersonation.enabled: "false" + hive.hdfs.wire-encryption.enabled: "true" + # HMS configuration + hive.metastore.authentication.type: KERBEROS + hive.metastore.client.principal: trino/trino.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + hive.metastore.client.keytab: /stackable/kerberos/keytab + hive.metastore.service.principal: hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + hive.metastore.thrift.impersonation.enabled: "false" + # By default, Hive views are executed with the RUN AS DEFINER security mode. Set the hive.hive-views.run-as-invoker catalog configuration property to true to use RUN AS INVOKER semantics. + # However, this does *not* work for Iceberg catalogs :/ (I asked on the Trino slack: https://trinodb.slack.com/archives/CJ6UC075E/p1711449384648869) + # hive.hive-views.run-as-invoker: "true" +{% endif %} diff --git a/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 index 2105ac96..7958931a 100644 --- a/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 +++ b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 @@ -8,7 +8,7 @@ spec: spec: containers: - name: create-iceberg-tables - image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev" command: - bash - -euo diff --git a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 index 99e8bd22..006c39e3 100644 --- a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 @@ -1,78 +1,6 @@ --- -apiVersion: nifi.stackable.tech/v1alpha1 -kind: NifiCluster -metadata: - name: nifi -spec: - image: -{% if test_scenario['values']['nifi-iceberg'].find(",") > 0 %} - custom: "{{ test_scenario['values']['nifi-iceberg'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['nifi-iceberg'].split(',')[0] }}" -{% else %} - custom: null - productVersion: "{{ test_scenario['values']['nifi-iceberg'] }}" -{% endif %} - pullPolicy: IfNotPresent - clusterConfig: - zookeeperConfigMapName: nifi-znode - listenerClass: external-unstable - authentication: - - authenticationClass: nifi-users - sensitiveProperties: - keySecret: nifi-sensitive-property-key - extraVolumes: - - name: hdfs-config - configMap: - name: hdfs -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - nodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - configOverrides: - nifi.properties: - # Quicker startup, and we only have a single node - nifi.cluster.flow.election.max.wait.time: 5 secs - jvmArgumentOverrides: - add: - # Needed for NiFi to trust the minio cert - - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 - - -Djavax.net.ssl.trustStorePassword=secret - - -Djavax.net.ssl.trustStoreType=PKCS12 - roleGroups: - default: - replicas: 1 ---- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: nifi-users -spec: - provider: - static: - userCredentialsSecret: - name: nifi-users ---- -apiVersion: v1 -kind: Secret -metadata: - name: nifi-users -stringData: - admin: adminadmin ---- -apiVersion: v1 -kind: Secret -metadata: - name: nifi-sensitive-property-key -stringData: - nifiSensitivePropsKey: mYsUp3rS3cr3tk3y ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperZnode -metadata: - name: nifi-znode -spec: - clusterRef: - name: zookeeper +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 50_nifi.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 new file mode 100644 index 00000000..4697a23e --- /dev/null +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -0,0 +1,111 @@ +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: nifi +spec: + image: +{% if test_scenario['values']['nifi-iceberg'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi-iceberg'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi-iceberg'].split(',')[0] }}" +{% else %} + custom: null + productVersion: "{{ test_scenario['values']['nifi-iceberg'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + zookeeperConfigMapName: nifi-znode + listenerClass: external-unstable + authentication: + - authenticationClass: nifi-users + sensitiveProperties: + keySecret: nifi-sensitive-property-key + extraVolumes: + - name: hdfs-config + configMap: + name: hdfs +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + nifi.properties: + # Quicker startup, and we only have a single node + nifi.cluster.flow.election.max.wait.time: 5 secs + jvmArgumentOverrides: + add: + # Needed for NiFi to trust the minio cert + - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 + - -Djavax.net.ssl.trustStorePassword=secret + - -Djavax.net.ssl.trustStoreType=PKCS12 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: &podOverrides + spec: + containers: + - name: nifi + env: + - name: KERBEROS_REALM + value: {{ test_scenario['values']['kerberos-realm'] }} + volumeMounts: + - name: kerberos + mountPath: /stackable/kerberos + # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`, + # but we can not influence the JVM arguments (edit: we can now) + - name: kerberos + mountPath: /etc/krb5.conf + subPath: krb5.conf + volumes: + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} + roleGroups: + default: + replicas: 1 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-users +stringData: + admin: adminadmin +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-znode +spec: + clusterRef: + name: zookeeper diff --git a/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 index 184675aa..079a2722 100644 --- a/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 +++ b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 @@ -8,8 +8,7 @@ spec: spec: containers: - name: check-iceberg-tables - # image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-latest'] }}-stackable0.0.0-dev" - image: "oci.stackable.tech/sdp/trino-cli:470-stackable0.0.0-dev" + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev" command: - bash - -euo diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml new file mode 100644 index 00000000..c893a079 --- /dev/null +++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml @@ -0,0 +1,47 @@ +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: nifi-interactive +spec: + image: + productVersion: 2.2.0 + clusterConfig: + authentication: + - authenticationClass: nifi-users + listenerClass: external-unstable + sensitiveProperties: + keySecret: nifi-interactive-sensitive-property-key + autoGenerate: true + zookeeperConfigMapName: nifi-interactive-znode + extraVolumes: + - name: nifi-processors + persistentVolumeClaim: + claimName: nifi-interactive-processors + nodes: + configOverrides: + nifi.properties: + nifi.nar.library.directory.myCustomLibs: "/stackable/userdata/nifi-processors/" + # Quicker startup + nifi.cluster.flow.election.max.wait.time: 3 secs + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-interactive-znode +spec: + clusterRef: + name: zookeeper +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nifi-interactive-processors +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 6b3e75c2..418c0c6f 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -18,9 +18,11 @@ dimensions: values: - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev - name: nifi-iceberg + # Not all NiFi versions support Iceberg with the same functionality! + # E.g. our own implementation started with NiFi 2.2.0 values: # - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev - - 2.0.0,oci.stackable.tech/sandbox/sbernauer/nifi:2.2.0-stackable0.0.0-dev-with-iceberg + - 2.0.0,oci.stackable.tech/sandbox/sbernauer/nifi:2.2.0-stackable0.0.0-dev - name: zookeeper values: - 3.9.2 @@ -28,15 +30,21 @@ dimensions: - name: zookeeper-latest values: - 3.9.3 - - name: hdfs-latest + - name: opa-l + values: + - 1.0.1 + - name: hdfs-l values: - 3.4.0 - - name: hive-latest + - name: hive-l values: - 4.0.1 - - name: trino-latest + - name: trino-l values: - "470" + - name: krb5 + values: + - 1.21.1 - name: ldap-use-tls values: - "false" @@ -45,6 +53,13 @@ dimensions: values: - "false" - "true" + - name: iceberg-use-kerberos + values: + - "false" + - "true" + - name: kerberos-realm + values: + - "PROD.MYCORP" - name: openshift values: - "false" @@ -101,10 +116,14 @@ tests: - name: iceberg dimensions: - nifi-iceberg + - opa-l - zookeeper-latest - - hdfs-latest - - hive-latest - - trino-latest + - hdfs-l + - hive-l + - trino-l + - krb5 + - iceberg-use-kerberos + - kerberos-realm - openshift suites: - name: nightly From 8ffd77914ae58edfa36bb0093587ee8544f0af16 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 7 May 2025 10:18:19 +0200 Subject: [PATCH 07/13] Add Kerberos test --- tests/templates/kuttl/iceberg/50_nifi.yaml.j2 | 43 + .../60-create-nifi-flow-configmap.yaml | 5 - .../60-create-nifi-flow-configmap.yaml.j2 | 10 + .../iceberg/60_nifi-flow-with-kerberos.json | 1144 +++++++++++++++++ ...son => 60_nifi-flow-without-kerberos.json} | 0 .../kuttl/iceberg/61-provision-nifi-flow.yaml | 2 +- tests/templates/kuttl/iceberg/README.md | 2 +- .../kuttl/iceberg/interactive-nifi.yaml | 43 +- 8 files changed, 1241 insertions(+), 8 deletions(-) delete mode 100644 tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml create mode 100644 tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json rename tests/templates/kuttl/iceberg/{60_nifi-flow.json => 60_nifi-flow-without-kerberos.json} (100%) diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 index 4697a23e..ac1d6ae3 100644 --- a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -24,6 +24,26 @@ spec: - name: hdfs-config configMap: name: hdfs +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + - name: hive-config + configMap: + name: nifi-hive-config + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -33,6 +53,10 @@ spec: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} configOverrides: nifi.properties: +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf +{% endif %} + # Quicker startup, and we only have a single node nifi.cluster.flow.election.max.wait.time: 5 secs jvmArgumentOverrides: @@ -109,3 +133,22 @@ metadata: spec: clusterRef: name: zookeeper +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi-hive-config +data: + hive-site.xml: | + + + hive.metastore.sasl.enabled + true + + + hive.metastore.kerberos.principal + hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + + +{% endif %} diff --git a/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml deleted file mode 100644 index fe92ec83..00000000 --- a/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: cat 60_nifi-flow.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin diff --git a/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 new file mode 100644 index 00000000..f6373a77 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + - script: cat 60_nifi-flow-with-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin +{% else %} + - script: cat 60_nifi-flow-without-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin + +{% endif %} diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json new file mode 100644 index 00000000..3be77142 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json @@ -0,0 +1,1144 @@ +{ + "externalControllerServices": {}, + "flowContents": { + "comments": "", + "componentType": "PROCESS_GROUP", + "connections": [ + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "c6381ef4-05d1-323a-9392-dac83fba33b3", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e", + "instanceIdentifier": "6872c492-329c-36fe-85b5-9593161ccbeb", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "name": "PutIceberg into S3", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "dd4c3b4d-7ad6-3ed3-b869-24a06f65a3e5", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "73b06526-5744-3aad-b148-995a910f28c5", + "instanceIdentifier": "c17f8b9d-ab2d-3797-82e2-deb7905f90ed", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "name": "PutIceberg into HDFS", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "name": "PutIceberg into HDFS", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "bc79d195-d4b2-3531-846e-77318e9f08c0", + "instanceIdentifier": "c197d602-e207-3d65-b7f7-cfc966a09089", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "5c168bce-0d60-372b-94f4-0690b880ce6c", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "79b3e396-3be1-3175-8c52-4f93347b1f9d", + "instanceIdentifier": "334a9821-8b6f-302c-a34d-51faa6bb36f1", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "name": "PutIceberg into HDFS", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "name": "PutIceberg into S3", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "11cf418f-4873-3913-b573-a2b07322b81d", + "instanceIdentifier": "01f6d8a0-cd07-36e0-b417-c757a727915b", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "77e72db2-fe65-31b7-a3b3-f9769051c8ac", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f", + "instanceIdentifier": "5558acd0-942d-34ed-8507-dca4bd42307b", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "name": "PutIceberg into S3", + "type": "PROCESSOR" + }, + "zIndex": 0 + } + ], + "controllerServices": [ + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-aws-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService" + }, + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "instanceIdentifier": "8bf71c50-5f81-3241-b983-28199b65d3d4", + "name": "AWSCredentialsProviderControllerService", + "properties": { + "anonymous-credentials": "false", + "Assume Role ARN": null, + "Assume Role Session Name": null, + "assume-role-external-id": null, + "assume-role-proxy-configuration-service": null, + "assume-role-ssl-context-service": null, + "assume-role-sts-endpoint": null, + "assume-role-sts-region": "us-west-2", + "assume-role-sts-signer-override": "Default Signature", + "Credentials File": null, + "custom-signer-class-name": null, + "custom-signer-module-location": null, + "default-credentials": "false", + "profile-name": null, + "Session Time": "3600", + "Access Key": "admin", + "Secret Key": "adminadmin" + }, + "propertyDescriptors": { + "Access Key": { + "displayName": "Access Key ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "Access Key", + "sensitive": true + }, + "anonymous-credentials": { + "displayName": "Use Anonymous Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "anonymous-credentials", + "sensitive": false + }, + "Assume Role ARN": { + "displayName": "Assume Role ARN", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role ARN", + "sensitive": false + }, + "Assume Role Session Name": { + "displayName": "Assume Role Session Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role Session Name", + "sensitive": false + }, + "assume-role-external-id": { + "displayName": "Assume Role External ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-external-id", + "sensitive": false + }, + "assume-role-proxy-configuration-service": { + "displayName": "Assume Role Proxy Configuration Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-proxy-configuration-service", + "sensitive": false + }, + "assume-role-ssl-context-service": { + "displayName": "Assume Role SSL Context Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-ssl-context-service", + "sensitive": false + }, + "assume-role-sts-endpoint": { + "displayName": "Assume Role STS Endpoint Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-endpoint", + "sensitive": false + }, + "assume-role-sts-region": { + "displayName": "Assume Role STS Region", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-region", + "sensitive": false + }, + "assume-role-sts-signer-override": { + "displayName": "Assume Role STS Signer Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-signer-override", + "sensitive": false + }, + "Credentials File": { + "displayName": "Credentials File", + "dynamic": false, + "identifiesControllerService": false, + "name": "Credentials File", + "resourceDefinition": { + "cardinality": "SINGLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "custom-signer-class-name": { + "displayName": "Custom Signer Class Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-class-name", + "sensitive": false + }, + "custom-signer-module-location": { + "displayName": "Custom Signer Module Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-module-location", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "DIRECTORY", + "FILE" + ] + }, + "sensitive": false + }, + "default-credentials": { + "displayName": "Use Default Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "default-credentials", + "sensitive": false + }, + "profile-name": { + "displayName": "Profile Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "profile-name", + "sensitive": false + }, + "Secret Key": { + "displayName": "Secret Access Key", + "dynamic": false, + "identifiesControllerService": false, + "name": "Secret Key", + "sensitive": true + }, + "Session Time": { + "displayName": "Assume Role Session Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "Session Time", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-record-serialization-services-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.serialization.RecordReaderFactory" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "instanceIdentifier": "b9cdca6d-40e9-3f45-bd6e-448e9c772985", + "name": "JsonTreeReader", + "properties": { + "Allow Comments": "false", + "Date Format": null, + "Max String Length": "20 MB", + "schema-access-strategy": "infer-schema", + "schema-application-strategy": "SELECTED_PART", + "schema-branch": null, + "schema-inference-cache": null, + "schema-name": "${schema.name}", + "schema-reference-reader": null, + "schema-registry": null, + "schema-text": "${avro.schema}", + "schema-version": null, + "starting-field-name": null, + "starting-field-strategy": "ROOT_NODE", + "Time Format": null, + "Timestamp Format": null + }, + "propertyDescriptors": { + "Allow Comments": { + "displayName": "Allow Comments", + "dynamic": false, + "identifiesControllerService": false, + "name": "Allow Comments", + "sensitive": false + }, + "Date Format": { + "displayName": "Date Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Date Format", + "sensitive": false + }, + "Max String Length": { + "displayName": "Max String Length", + "dynamic": false, + "identifiesControllerService": false, + "name": "Max String Length", + "sensitive": false + }, + "schema-access-strategy": { + "displayName": "Schema Access Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-access-strategy", + "sensitive": false + }, + "schema-application-strategy": { + "displayName": "Schema Application Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-application-strategy", + "sensitive": false + }, + "schema-branch": { + "displayName": "Schema Branch", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-branch", + "sensitive": false + }, + "schema-inference-cache": { + "displayName": "Schema Inference Cache", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-inference-cache", + "sensitive": false + }, + "schema-name": { + "displayName": "Schema Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-name", + "sensitive": false + }, + "schema-reference-reader": { + "displayName": "Schema Reference Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-reference-reader", + "sensitive": false + }, + "schema-registry": { + "displayName": "Schema Registry", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-registry", + "sensitive": false + }, + "schema-text": { + "displayName": "Schema Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-text", + "sensitive": false + }, + "schema-version": { + "displayName": "Schema Version", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-version", + "sensitive": false + }, + "starting-field-name": { + "displayName": "Starting Field Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-name", + "sensitive": false + }, + "starting-field-strategy": { + "displayName": "Starting Field Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-strategy", + "sensitive": false + }, + "Time Format": { + "displayName": "Time Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Time Format", + "sensitive": false + }, + "Timestamp Format": { + "displayName": "Timestamp Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Timestamp Format", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.json.JsonTreeReader" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-kerberos-user-service-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.kerberos.SelfContainedKerberosUserService" + }, + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.kerberos.KerberosUserService" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "79d1509c-17ab-3c2a-a431-51324f50e829", + "instanceIdentifier": "a5bbd59e-0196-1000-0000-0000560be512", + "name": "KerberosKeytabUserService", + "properties": { + "Kerberos Keytab": "/stackable/userdata/kerberos/keytab", + "Kerberos Principal": "nifi/nifi.${NAMESPACE}.svc.cluster.local@PROD.MYCORP" + }, + "propertyDescriptors": { + "Kerberos Keytab": { + "displayName": "Kerberos Keytab", + "dynamic": false, + "identifiesControllerService": false, + "name": "Kerberos Keytab", + "resourceDefinition": { + "cardinality": "SINGLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "Kerberos Principal": { + "displayName": "Kerberos Principal", + "dynamic": false, + "identifiesControllerService": false, + "name": "Kerberos Principal", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.kerberos.KerberosKeytabUserService" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-services-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-iceberg-services-api-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "instanceIdentifier": "e2c4259a-d4e0-3be9-af1c-c361796e6a0f", + "name": "IcebergHiveCatalogService", + "properties": { + "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml,/stackable/userdata/hive-config/hive-site.xml", + "hive-metastore-uri": "thrift://hive:9083", + "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-path-style-access": "true", + "warehouse-location": "s3a://demo/lakehouse" + }, + "propertyDescriptors": { + "AWS Credentials Provider service": { + "displayName": "AWS Credentials Provider Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "AWS Credentials Provider service", + "sensitive": false + }, + "hadoop-config-resources": { + "displayName": "Hadoop Configuration Resources", + "dynamic": false, + "identifiesControllerService": false, + "name": "hadoop-config-resources", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "hive-metastore-uri": { + "displayName": "Hive Metastore URI", + "dynamic": false, + "identifiesControllerService": false, + "name": "hive-metastore-uri", + "sensitive": false + }, + "s3-endpoint": { + "displayName": "S3 endpoint", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-endpoint", + "sensitive": false + }, + "s3-path-style-access": { + "displayName": "S3 path style access", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-path-style-access", + "sensitive": false + }, + "warehouse-location": { + "displayName": "Default Warehouse Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "warehouse-location", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "tech.stackable.nifi.services.iceberg.IcebergHiveCatalogService" + } + ], + "defaultBackPressureDataSizeThreshold": "1 GB", + "defaultBackPressureObjectThreshold": 10000, + "defaultFlowFileExpiration": "0 sec", + "executionEngine": "INHERITED", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "funnels": [ + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "5c168bce-0d60-372b-94f4-0690b880ce6c", + "position": { + "x": -880.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "dd4c3b4d-7ad6-3ed3-b869-24a06f65a3e5", + "position": { + "x": -1344.0, + "y": -432.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "77e72db2-fe65-31b7-a3b3-f9769051c8ac", + "position": { + "x": -328.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "c6381ef4-05d1-323a-9392-dac83fba33b3", + "position": { + "x": 160.0, + "y": -432.0 + } + } + ], + "identifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "inputPorts": [], + "instanceIdentifier": "a5a6f0ac-0196-1000-ffff-fffffaba5637", + "labels": [], + "maxConcurrentTasks": 1, + "name": "Iceberg Test", + "outputPorts": [], + "position": { + "x": 100.0, + "y": 10.0 + }, + "processGroups": [], + "processors": [ + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg into HDFS", + "penaltyDuration": "30 sec", + "position": { + "x": -1032.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "hdfs", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "ORC", + "kerberos-user-service": "79d1509c-17ab-3c2a-a431-51324f50e829", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "kerberos-user-service": { + "displayName": "Kerberos User Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "kerberos-user-service", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg into S3", + "penaltyDuration": "30 sec", + "position": { + "x": -480.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "s3", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "PARQUET", + "kerberos-user-service": "79d1509c-17ab-3c2a-a431-51324f50e829", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "kerberos-user-service": { + "displayName": "Kerberos User Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "kerberos-user-service", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3", + "maxBackoffPeriod": "10 mins", + "name": "GenerateFlowFile", + "penaltyDuration": "30 sec", + "position": { + "x": -744.0, + "y": -704.0 + }, + "properties": { + "Batch Size": "1", + "character-set": "UTF-8", + "Data Format": "Text", + "File Size": "0B", + "generate-ff-custom-text": "{\"hello\": \"world from NiFi :)\"}", + "mime-type": null, + "Unique FlowFiles": "false" + }, + "propertyDescriptors": { + "Batch Size": { + "displayName": "Batch Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "Batch Size", + "sensitive": false + }, + "character-set": { + "displayName": "Character Set", + "dynamic": false, + "identifiesControllerService": false, + "name": "character-set", + "sensitive": false + }, + "Data Format": { + "displayName": "Data Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Data Format", + "sensitive": false + }, + "File Size": { + "displayName": "File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "File Size", + "sensitive": false + }, + "generate-ff-custom-text": { + "displayName": "Custom Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "generate-ff-custom-text", + "sensitive": false + }, + "mime-type": { + "displayName": "Mime Type", + "dynamic": false, + "identifiesControllerService": false, + "name": "mime-type", + "sensitive": false + }, + "Unique FlowFiles": { + "displayName": "Unique FlowFiles", + "dynamic": false, + "identifiesControllerService": false, + "name": "Unique FlowFiles", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "1 min", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "yieldDuration": "1 sec" + } + ], + "remoteProcessGroups": [], + "scheduledState": "ENABLED", + "statelessFlowTimeout": "1 min" + }, + "flowEncodingVersion": "1.0", + "latest": false, + "parameterContexts": {}, + "parameterProviders": {} +} \ No newline at end of file diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json similarity index 100% rename from tests/templates/kuttl/iceberg/60_nifi-flow.json rename to tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml index 09cd08fa..89ea6bba 100644 --- a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -98,7 +98,7 @@ data: print("Process group uploaded") # As they are started in the wrong order :D we need to retry - max_retries = 3 + max_retries = 5 for _ in range(max_retries): controllers = list_all_controllers(pg_id) print(f"Found {len(controllers)} controllers") diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md index da3e9d1d..b856148c 100644 --- a/tests/templates/kuttl/iceberg/README.md +++ b/tests/templates/kuttl/iceberg/README.md @@ -4,7 +4,7 @@ The file `60_nifi-flow.json` was exported from the NiFi UI. TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier. -Notable the following diff has been made: +Notable the following diff has been made (may not be up to date!): ```diff diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml index c893a079..d21fff01 100644 --- a/tests/templates/kuttl/iceberg/interactive-nifi.yaml +++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml @@ -17,12 +17,36 @@ spec: - name: nifi-processors persistentVolumeClaim: claimName: nifi-interactive-processors + - name: hdfs-config + configMap: + name: hdfs + - name: nifi-interactive-hive-site + configMap: + name: nifi-interactive-hive-site + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-kuttl-test-united-pheasant + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech nodes: configOverrides: nifi.properties: - nifi.nar.library.directory.myCustomLibs: "/stackable/userdata/nifi-processors/" + nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ + nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf # Quicker startup nifi.cluster.flow.election.max.wait.time: 3 secs + envOverrides: + KERBEROS_REALM: PROD.MYCORP roleGroups: default: replicas: 1 @@ -45,3 +69,20 @@ spec: resources: requests: storage: 10Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi-interactive-hive-site +data: + hive-site.xml: | + + + hive.metastore.kerberos.principal + hive/hive.kuttl-test-united-pheasant.svc.cluster.local@MY.CORP + + + hive.metastore.sasl.enabled + true + + From f9258840e0ba5c6bc39460e9947bcb79ed054b34 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 7 May 2025 13:56:11 +0200 Subject: [PATCH 08/13] Use nightly image --- tests/test-definition.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 418c0c6f..711c1055 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -21,8 +21,7 @@ dimensions: # Not all NiFi versions support Iceberg with the same functionality! # E.g. our own implementation started with NiFi 2.2.0 values: - # - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev - - 2.0.0,oci.stackable.tech/sandbox/sbernauer/nifi:2.2.0-stackable0.0.0-dev + - 2.2.0 # oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev - name: zookeeper values: - 3.9.2 From dfefa4d73beef34b308bad6a9dea5f2251c287ef Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 8 May 2025 10:06:43 +0200 Subject: [PATCH 09/13] linter --- tests/templates/kuttl/iceberg/34_trino.yaml.j2 | 2 +- tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json | 2 +- .../templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json | 2 +- tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 index 86491360..e5c1e80c 100644 --- a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -20,7 +20,7 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - coordinators: + coordinators: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json index 3be77142..a5f773f1 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json @@ -1141,4 +1141,4 @@ "latest": false, "parameterContexts": {}, "parameterProviders": {} -} \ No newline at end of file +} diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json index 2c6ed7d1..c63158b9 100644 --- a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json @@ -1065,4 +1065,4 @@ "latest": false, "parameterContexts": {}, "parameterProviders": {} -} \ No newline at end of file +} diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml index 89ea6bba..7ab671ce 100644 --- a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -68,7 +68,7 @@ data: print("Logged in") filename = "/tmp/nifi-flow/nifi-flow.json" - + pg_id = get_root_pg_id() print(f"Got root process group id: {pg_id}") From 841e48423b3a347118c626694970381e44f8b0d1 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 8 May 2025 10:16:13 +0200 Subject: [PATCH 10/13] Update Iceberg docs --- .../writing-to-iceberg-tables.adoc | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc index a026a5b6..9d8968a0 100644 --- a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc +++ b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc @@ -2,13 +2,27 @@ :description: Write to Apache Iceberg tables in NiFi using the PutIceberg processor. Supports integration with S3 and Hive Metastore for scalable data handling. :iceberg: https://iceberg.apache.org/ -WARNING: In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed]. - {iceberg}[Apache Iceberg] is a high-performance format for huge analytic tables. Iceberg brings the reliability and simplicity of SQL tables to big data, while making it possible for engines like Spark, Trino, Flink, Presto, Hive and Impala to safely work with the same tables, at the same time. NiFi supports a `PutIceberg` processor to add rows to an existing Iceberg table https://issues.apache.org/jira/browse/NIFI-10442[starting from version 1.19.0]. -As of NiFi version `1.23.1` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform). +As of NiFi version `2.4.0` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform). + +== NiFi 2.x + +In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed] from upstream NiFi. + +We forked the `nifi-iceberg-bundle` and made it available at https://github.com/stackabletech/nifi-iceberg-bundle. +Starting with SDP 25.7, we have added the necessary bundle to NiFi by default, you don't need to explicitly add Iceberg support to the Stackable NiFi. + +Please read on https://github.com/stackabletech/nifi-iceberg-bundle[its documentation] on how to ingest data into Iceberg tables. +You don't need any special configs on the `NiFiCluster` in case you are using S3 and no Kerberos. + +HDFS and Kerberos are also supported, please have a look at the https://github.com/stackabletech/nifi-operator/tree/main/tests/templates/kuttl/iceberg[Iceberg integration test] for that. + +== NiFi 1.x + +Starting with `1.19.0`, NiFi supports writing to Iceberg tables. The following example shows an example NiFi setup using the Iceberg integration. From a689cdbb6b6b8c78a78d4e7c83f7de8f4bb9e471 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 15 May 2025 10:16:43 +0200 Subject: [PATCH 11/13] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0497f782..1ec44e88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. +- Add test for Apache Iceberg integration ([#785]). ### Changed @@ -39,6 +40,7 @@ All notable changes to this project will be documented in this file. [#774]: https://github.com/stackabletech/nifi-operator/pull/774 [#776]: https://github.com/stackabletech/nifi-operator/pull/776 [#782]: https://github.com/stackabletech/nifi-operator/pull/782 +[#785]: https://github.com/stackabletech/nifi-operator/pull/785 [#787]: https://github.com/stackabletech/nifi-operator/pull/787 [#789]: https://github.com/stackabletech/nifi-operator/pull/789 From ca3d817a5badf502f120a2f3caef7c8c77e3e775 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 21 May 2025 09:23:32 +0200 Subject: [PATCH 12/13] Small bumps --- tests/test-definition.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 946968d3..55e51bc1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -30,10 +30,10 @@ dimensions: - 3.9.3 - name: opa-l values: - - 1.0.1 + - 1.4.2 - name: hdfs-l values: - - 3.4.0 + - 3.4.1 - name: hive-l values: - 4.0.1 From 8b612cf54e7e07ddb814168359fd427bf5e78198 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 21 May 2025 09:28:53 +0200 Subject: [PATCH 13/13] Update docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- .../nifi/pages/usage_guide/writing-to-iceberg-tables.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc index 9d8968a0..13048014 100644 --- a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc +++ b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc @@ -8,7 +8,7 @@ Iceberg brings the reliability and simplicity of SQL tables to big data, while m NiFi supports a `PutIceberg` processor to add rows to an existing Iceberg table https://issues.apache.org/jira/browse/NIFI-10442[starting from version 1.19.0]. As of NiFi version `2.4.0` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform). -== NiFi 2.x +== NiFi 2 In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed] from upstream NiFi. @@ -20,7 +20,7 @@ You don't need any special configs on the `NiFiCluster` in case you are using S3 HDFS and Kerberos are also supported, please have a look at the https://github.com/stackabletech/nifi-operator/tree/main/tests/templates/kuttl/iceberg[Iceberg integration test] for that. -== NiFi 1.x +== NiFi 1 Starting with `1.19.0`, NiFi supports writing to Iceberg tables.