diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a98826..243c4cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. - NiFi 2.x now supports storing cluster state in Kubernetes instead of ZooKeeper ([#775]). - Add test for Apache Iceberg integration ([#785]). +- Add support for custom components via git-sync ([#793]). ### Changed @@ -50,6 +51,7 @@ All notable changes to this project will be documented in this file. [#785]: https://github.com/stackabletech/nifi-operator/pull/785 [#787]: https://github.com/stackabletech/nifi-operator/pull/787 [#789]: https://github.com/stackabletech/nifi-operator/pull/789 +[#793]: https://github.com/stackabletech/nifi-operator/pull/793 [#799]: https://github.com/stackabletech/nifi-operator/pull/799 [#801]: https://github.com/stackabletech/nifi-operator/pull/801 diff --git a/Cargo.lock b/Cargo.lock index 92e49bf7..06883a8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,40 +173,13 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ - "axum-core 0.5.2", + "axum-core", "bytes", "form_urlencoded", "futures-util", @@ -216,7 +189,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -234,26 +207,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.2" @@ -602,6 +555,26 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "digest" version = "0.10.7" @@ -1467,33 +1440,33 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c75b990324f09bef15e791606b7b7a296d02fc88a344f6eba9390970a870ad5" +checksum = "aa60a41b57ae1a0a071af77dbcf89fc9819cfe66edaf2beeb204c34459dcf0b2" dependencies = [ "base64 0.22.1", "chrono", "schemars", "serde", - "serde-value", "serde_json", ] [[package]] name = "k8s-version" -version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +version = "0.1.3" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ "darling", "regex", + "serde", "snafu 0.8.5", ] [[package]] name = "kube" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4eb20010536b48abe97fec37d23d43069bcbe9686adcf9932202327bc5ca6e" +checksum = "1b49c39074089233c2bb7b1791d1b6c06c84dbab26757491fad9d233db0d432f" dependencies = [ "k8s-openapi", "kube-client", @@ -1504,9 +1477,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc2ed952042df20d15ac2fe9614d0ec14b6118eab89633985d4b36e688dccf1" +checksum = "e199797b1b08865041c9c698f0d11a91de0a8143e808b71e250cd4a1d7ce2b9f" dependencies = [ "base64 0.22.1", "bytes", @@ -1541,11 +1514,12 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff0d0793db58e70ca6d689489183816cb3aa481673e7433dc618cf7e8007c675" +checksum = "1bdefbba89dea2d99ea822a1d7cd6945535efbfb10b790056ee9284bf9e698e7" dependencies = [ "chrono", + "derive_more", "form_urlencoded", "http", "json-patch", @@ -1559,9 +1533,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c562f58dc9f7ca5feac8a6ee5850ca221edd6f04ce0dd2ee873202a88cd494c9" +checksum = "8e609a3633689a50869352a3c16e01d863b6137863c80eeb038383d5ab9f83bf" dependencies = [ "darling", "proc-macro2", @@ -1573,14 +1547,13 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f34cfab9b4bd8633062e0e85edb81df23cb09f159f2e31c60b069ae826ffdc" +checksum = "1d4bd8a4554786f8f9a87bfa977fb7dbaa1d7f102a30477338b044b65de29d8e" dependencies = [ "ahash", "async-broadcast", "async-stream", - "async-trait", "backon", "educe", "futures 0.3.31", @@ -1666,12 +1639,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -1758,9 +1725,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" -version = "0.28.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" +checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" dependencies = [ "futures-core", "futures-sink", @@ -1772,9 +1739,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.28.1" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c513c7af3bec30113f3d4620134ff923295f1e9c580fda2b8abe0831f925ddc0" +checksum = "e716f864eb23007bdd9dc4aec381e188a1cee28eecf22066772b5fd822b9727d" dependencies = [ "opentelemetry", "tracing", @@ -1784,9 +1751,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253" +checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" dependencies = [ "async-trait", "bytes", @@ -1798,11 +1765,10 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91" +checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" dependencies = [ - "async-trait", "futures-core", "http", "opentelemetry", @@ -1819,9 +1785,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d" +checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1831,18 +1797,17 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" +checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" dependencies = [ - "async-trait", "futures-channel", "futures-executor", "futures-util", "glob", "opentelemetry", "percent-encoding", - "rand 0.8.5", + "rand 0.9.0", "serde_json", "thiserror 2.0.12", "tokio", @@ -2700,8 +2665,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.92.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +version = "0.93.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ "chrono", "clap", @@ -2738,7 +2703,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ "darling", "proc-macro2", @@ -2749,7 +2714,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ "kube", "semver", @@ -2761,9 +2726,9 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ - "axum 0.8.3", + "axum", "clap", "futures-util", "opentelemetry", @@ -2784,15 +2749,20 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.7.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ + "k8s-version", + "schemars", + "serde", + "serde_json", + "serde_yaml", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.7.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#e0309b9f85413846984ff16f6e522d0f720dee43" dependencies = [ "convert_case", "darling", @@ -3059,13 +3029,10 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", "flate2", - "h2", "http", "http-body", "http-body-util", @@ -3075,7 +3042,6 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "socket2", "tokio", "tokio-stream", "tower 0.4.13", @@ -3209,9 +3175,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721f2d2569dce9f3dfbbddee5906941e953bfcdf736a62da3377f5751650cc36" +checksum = "fd8e764bd6f5813fd8bebc3117875190c5b0415be8f7f8059bffb6ecd979c444" dependencies = [ "js-sys", "once_cell", diff --git a/Cargo.nix b/Cargo.nix index 934bf0b7..d8ead010 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -516,123 +516,7 @@ rec { ]; }; - "axum 0.7.9" = rec { - crateName = "axum"; - version = "0.7.9"; - edition = "2021"; - sha256 = "07z7wqczi9i8xb4460rvn39p4wjqwr32hx907crd1vwb2fy8ijpd"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "axum-core"; - packageId = "axum-core 0.4.5"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "http"; - packageId = "http"; - } - { - name = "http-body"; - packageId = "http-body"; - } - { - name = "http-body-util"; - packageId = "http-body-util"; - } - { - name = "itoa"; - packageId = "itoa"; - } - { - name = "matchit"; - packageId = "matchit 0.7.3"; - } - { - name = "memchr"; - packageId = "memchr"; - } - { - name = "mime"; - packageId = "mime"; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "rustversion"; - packageId = "rustversion"; - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "sync_wrapper"; - packageId = "sync_wrapper"; - } - { - name = "tower"; - packageId = "tower 0.5.2"; - usesDefaultFeatures = false; - features = [ "util" ]; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - ]; - devDependencies = [ - { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; - } - { - name = "tower"; - packageId = "tower 0.5.2"; - rename = "tower"; - features = [ "util" "timeout" "limit" "load-shed" "steer" "filter" ]; - } - ]; - features = { - "__private_docs" = [ "axum-core/__private_docs" "tower/full" "dep:tower-http" ]; - "default" = [ "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; - "form" = [ "dep:serde_urlencoded" ]; - "http1" = [ "dep:hyper" "hyper?/http1" "hyper-util?/http1" ]; - "http2" = [ "dep:hyper" "hyper?/http2" "hyper-util?/http2" ]; - "json" = [ "dep:serde_json" "dep:serde_path_to_error" ]; - "macros" = [ "dep:axum-macros" ]; - "multipart" = [ "dep:multer" ]; - "query" = [ "dep:serde_urlencoded" ]; - "tokio" = [ "dep:hyper-util" "dep:tokio" "tokio/net" "tokio/rt" "tower/make" "tokio/macros" ]; - "tower-log" = [ "tower/log" ]; - "tracing" = [ "dep:tracing" "axum-core/tracing" ]; - "ws" = [ "dep:hyper" "tokio" "dep:tokio-tungstenite" "dep:sha1" "dep:base64" ]; - }; - }; - "axum 0.8.3" = rec { + "axum" = rec { crateName = "axum"; version = "0.8.3"; edition = "2021"; @@ -640,7 +524,7 @@ rec { dependencies = [ { name = "axum-core"; - packageId = "axum-core 0.5.2"; + packageId = "axum-core"; } { name = "bytes"; @@ -686,7 +570,7 @@ rec { } { name = "matchit"; - packageId = "matchit 0.8.4"; + packageId = "matchit"; } { name = "memchr"; @@ -809,80 +693,9 @@ rec { "tracing" = [ "dep:tracing" "axum-core/tracing" ]; "ws" = [ "dep:hyper" "tokio" "dep:tokio-tungstenite" "dep:sha1" "dep:base64" ]; }; - resolvedDefaultFeatures = [ "default" "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; - }; - "axum-core 0.4.5" = rec { - crateName = "axum-core"; - version = "0.4.5"; - edition = "2021"; - sha256 = "16b1496c4gm387q20hkv5ic3k5bd6xmnvk50kwsy6ymr8rhvvwh9"; - libName = "axum_core"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "http"; - packageId = "http"; - } - { - name = "http-body"; - packageId = "http-body"; - } - { - name = "http-body-util"; - packageId = "http-body-util"; - } - { - name = "mime"; - packageId = "mime"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "rustversion"; - packageId = "rustversion"; - } - { - name = "sync_wrapper"; - packageId = "sync_wrapper"; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - ]; - devDependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - ]; - features = { - "__private_docs" = [ "dep:tower-http" ]; - "tracing" = [ "dep:tracing" ]; - }; + resolvedDefaultFeatures = [ "default" "form" "http1" "http2" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; }; - "axum-core 0.5.2" = rec { + "axum-core" = rec { crateName = "axum-core"; version = "0.5.2"; edition = "2021"; @@ -1901,6 +1714,94 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "powerfmt" "std" ]; }; + "derive_more" = rec { + crateName = "derive_more"; + version = "2.0.1"; + edition = "2021"; + sha256 = "0y3n97cc7rsvgnj211p92y1ppzh6jzvq5kvk6340ghkhfp7l4ch9"; + authors = [ + "Jelte Fennema " + ]; + dependencies = [ + { + name = "derive_more-impl"; + packageId = "derive_more-impl"; + } + ]; + features = { + "add" = [ "derive_more-impl/add" ]; + "add_assign" = [ "derive_more-impl/add_assign" ]; + "as_ref" = [ "derive_more-impl/as_ref" ]; + "constructor" = [ "derive_more-impl/constructor" ]; + "debug" = [ "derive_more-impl/debug" ]; + "default" = [ "std" ]; + "deref" = [ "derive_more-impl/deref" ]; + "deref_mut" = [ "derive_more-impl/deref_mut" ]; + "display" = [ "derive_more-impl/display" ]; + "error" = [ "derive_more-impl/error" ]; + "from" = [ "derive_more-impl/from" ]; + "from_str" = [ "derive_more-impl/from_str" ]; + "full" = [ "add" "add_assign" "as_ref" "constructor" "debug" "deref" "deref_mut" "display" "error" "from" "from_str" "index" "index_mut" "into" "into_iterator" "is_variant" "mul" "mul_assign" "not" "sum" "try_from" "try_into" "try_unwrap" "unwrap" ]; + "index" = [ "derive_more-impl/index" ]; + "index_mut" = [ "derive_more-impl/index_mut" ]; + "into" = [ "derive_more-impl/into" ]; + "into_iterator" = [ "derive_more-impl/into_iterator" ]; + "is_variant" = [ "derive_more-impl/is_variant" ]; + "mul" = [ "derive_more-impl/mul" ]; + "mul_assign" = [ "derive_more-impl/mul_assign" ]; + "not" = [ "derive_more-impl/not" ]; + "sum" = [ "derive_more-impl/sum" ]; + "testing-helpers" = [ "derive_more-impl/testing-helpers" "dep:rustc_version" ]; + "try_from" = [ "derive_more-impl/try_from" ]; + "try_into" = [ "derive_more-impl/try_into" ]; + "try_unwrap" = [ "derive_more-impl/try_unwrap" ]; + "unwrap" = [ "derive_more-impl/unwrap" ]; + }; + resolvedDefaultFeatures = [ "default" "from" "std" ]; + }; + "derive_more-impl" = rec { + crateName = "derive_more-impl"; + version = "2.0.1"; + edition = "2021"; + sha256 = "1wqxcb7d5lzvpplz9szp4rwy1r23f5wmixz0zd2vcjscqknji9mx"; + procMacro = true; + libName = "derive_more_impl"; + authors = [ + "Jelte Fennema " + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.100"; + } + ]; + features = { + "as_ref" = [ "syn/extra-traits" "syn/visit" ]; + "debug" = [ "syn/extra-traits" "dep:unicode-xid" ]; + "display" = [ "syn/extra-traits" "dep:unicode-xid" ]; + "error" = [ "syn/extra-traits" ]; + "from" = [ "syn/extra-traits" ]; + "full" = [ "add" "add_assign" "as_ref" "constructor" "debug" "deref" "deref_mut" "display" "error" "from" "from_str" "index" "index_mut" "into" "into_iterator" "is_variant" "mul" "mul_assign" "not" "sum" "try_from" "try_into" "try_unwrap" "unwrap" ]; + "into" = [ "syn/extra-traits" ]; + "is_variant" = [ "dep:convert_case" ]; + "mul" = [ "syn/extra-traits" ]; + "mul_assign" = [ "syn/extra-traits" ]; + "not" = [ "syn/extra-traits" ]; + "testing-helpers" = [ "dep:rustc_version" ]; + "try_into" = [ "syn/extra-traits" ]; + "try_unwrap" = [ "dep:convert_case" ]; + "unwrap" = [ "dep:convert_case" ]; + }; + resolvedDefaultFeatures = [ "default" "from" ]; + }; "digest" = rec { crateName = "digest"; version = "0.10.7"; @@ -3760,7 +3661,7 @@ rec { "tokio" = [ "dep:tokio" "tokio/net" "tokio/rt" "tokio/time" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "client" "client-legacy" "default" "http1" "http2" "server" "server-auto" "service" "tokio" ]; + resolvedDefaultFeatures = [ "client" "client-legacy" "default" "http1" "http2" "server" "service" "tokio" "tracing" ]; }; "iana-time-zone" = rec { crateName = "iana-time-zone"; @@ -4613,10 +4514,10 @@ rec { }; "k8s-openapi" = rec { crateName = "k8s-openapi"; - version = "0.24.0"; + version = "0.25.0"; edition = "2021"; - links = "k8s-openapi-0.24.0"; - sha256 = "1m8ahw59g44kp9p4yd4ar0px15m2nyvhc5krbvqvw2ag6a8bjx9c"; + links = "k8s-openapi-0.25.0"; + sha256 = "1cphvicl9hq4nbp2pbzdcvz9r0f9kzwbqzgp383hl6mfawds8q5a"; libName = "k8s_openapi"; authors = [ "Arnav Singh " @@ -4645,11 +4546,6 @@ rec { packageId = "serde"; usesDefaultFeatures = false; } - { - name = "serde-value"; - packageId = "serde-value"; - usesDefaultFeatures = false; - } { name = "serde_json"; packageId = "serde_json"; @@ -4658,21 +4554,22 @@ rec { } ]; features = { - "earliest" = [ "v1_28" ]; - "latest" = [ "v1_32" ]; + "default" = [ "std" ]; + "earliest" = [ "v1_30" ]; + "latest" = [ "v1_33" ]; "schemars" = [ "dep:schemars" ]; }; - resolvedDefaultFeatures = [ "schemars" "v1_32" ]; + resolvedDefaultFeatures = [ "schemars" "v1_33" ]; }; "k8s-version" = rec { crateName = "k8s-version"; - version = "0.1.2"; + version = "0.1.3"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; libName = "k8s_version"; authors = [ @@ -4688,6 +4585,12 @@ rec { name = "regex"; packageId = "regex"; } + { + name = "serde"; + packageId = "serde"; + optional = true; + features = [ "derive" ]; + } { name = "snafu"; packageId = "snafu 0.8.5"; @@ -4695,14 +4598,15 @@ rec { ]; features = { "darling" = [ "dep:darling" ]; + "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "darling" ]; + resolvedDefaultFeatures = [ "darling" "serde" ]; }; "kube" = rec { crateName = "kube"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "0vnaqmxk40i2jgwxqsk8x75rn1j37p93gv3zx6mlhssk200b4kls"; + sha256 = "0bs31pdk7lnrza8p8x96mgdq8v60nv8r25vvpg1374h8fj8c6j8v"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4773,9 +4677,9 @@ rec { }; "kube-client" = rec { crateName = "kube-client"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "1wfciml6xcylhlwn72dbiq8vc57cs0a9dzn2bb8j1ps242ayvhkz"; + sha256 = "17rbrvbs3m0c4lgbf2788f0hmpli3b8z1666r50m11h83dxpk6g1"; libName = "kube_client"; authors = [ "clux " @@ -4858,7 +4762,7 @@ rec { name = "hyper-util"; packageId = "hyper-util"; optional = true; - features = [ "client" "client-legacy" "http1" "tokio" ]; + features = [ "client" "client-legacy" "http1" "tokio" "tracing" ]; } { name = "jsonpath-rust"; @@ -5014,9 +4918,9 @@ rec { }; "kube-core" = rec { crateName = "kube-core"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "0xf60y07xkqqqqyl7rvk2r4amcvch61r2j49ssk0rrsqvf9hf3gz"; + sha256 = "1rwqwvwlna79dq2r1dqhzgxmwls5d76xg892m2gdk8nyi6xgpphv"; libName = "kube_core"; authors = [ "clux " @@ -5030,6 +4934,11 @@ rec { usesDefaultFeatures = false; features = [ "now" ]; } + { + name = "derive_more"; + packageId = "derive_more"; + features = [ "from" ]; + } { name = "form_urlencoded"; packageId = "form_urlencoded"; @@ -5091,9 +5000,9 @@ rec { }; "kube-derive" = rec { crateName = "kube-derive"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "1jclsj6ah0ijhzpd43ff0ipxs7i2r985ivm6r3m5zjppr66zaqn5"; + sha256 = "1gw3kymxb0w30gmhxj33g09vcqyq05pc38sjjf3516k86cv9lq4f"; procMacro = true; libName = "kube_derive"; authors = [ @@ -5140,9 +5049,9 @@ rec { }; "kube-runtime" = rec { crateName = "kube-runtime"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "1p7z4vl9l1hbqqqjx7qmkyq3rwhxp3nqa3if0qrqdgdlp7x4rww8"; + sha256 = "13lxw9fvci5h71rlfc1a21zivanvnxzrgykvm3wzi1j7anjdhjqx"; libName = "kube_runtime"; authors = [ "clux " @@ -5162,10 +5071,6 @@ rec { name = "async-stream"; packageId = "async-stream"; } - { - name = "async-trait"; - packageId = "async-trait"; - } { name = "backon"; packageId = "backon"; @@ -5460,19 +5365,7 @@ rec { ]; }; - "matchit 0.7.3" = rec { - crateName = "matchit"; - version = "0.7.3"; - edition = "2021"; - sha256 = "156bgdmmlv4crib31qhgg49nsjk88dxkdqp80ha2pk2rk6n6ax0f"; - authors = [ - "Ibraheem Ahmed " - ]; - features = { - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "matchit 0.8.4" = rec { + "matchit" = rec { crateName = "matchit"; version = "0.8.4"; edition = "2021"; @@ -5715,9 +5608,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.28.0"; + version = "0.29.1"; edition = "2021"; - sha256 = "09k43sgaarw3zx5j434ngq1canpcjibsbxaqqa8dyp0acxxncvi3"; + sha256 = "0v6ijlxs486yip2zbjdpgqc525q8l8k9s8ddz6b4ixvm4xz271wy"; dependencies = [ { name = "futures-core"; @@ -5753,7 +5646,8 @@ rec { } ]; features = { - "default" = [ "trace" "metrics" "logs" "internal-logs" ]; + "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; @@ -5761,16 +5655,16 @@ rec { "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; - "trace" = [ "pin-project-lite" "futures-sink" "futures-core" "thiserror" ]; + "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.28.1"; + version = "0.29.1"; edition = "2021"; - sha256 = "1h6x4pwk225yi8mxl3sqkhg5ya93z57i68267lzi2c7c7fpwf4y5"; + sha256 = "0zbjp4idhprbfxk21wpcivicx8c8w60w7bn4kpfpn013xdjgh5p7"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -5797,11 +5691,17 @@ rec { } ]; devDependencies = [ + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + features = [ "std" ]; + } { name = "tracing-subscriber"; packageId = "tracing-subscriber"; usesDefaultFeatures = false; - features = [ "registry" "std" "env-filter" ]; + features = [ "env-filter" "registry" "std" "fmt" ]; } ]; features = { @@ -5815,9 +5715,9 @@ rec { }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "0lv2sbsdr7b8bxnly92zzhlm1wzjbynib1xlkw9hs0qh56pkz1m8"; + sha256 = "1vf86z9d4dr9msck3k2xan9w5k35rfk9bylhpnav9d97p0rapms6"; libName = "opentelemetry_http"; dependencies = [ { @@ -5866,15 +5766,11 @@ rec { }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "148xq13ar11bvmk7pxbslrhh5pgf40bv83n6dlysigj1dm613vsv"; + sha256 = "0mjnx260qn4x1p9pyip35m7764kkszn087f0f6xcq5k9w07p56fq"; libName = "opentelemetry_otlp"; dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } { name = "futures-core"; packageId = "futures-core"; @@ -5949,6 +5845,12 @@ rec { usesDefaultFeatures = false; features = [ "macros" "rt-multi-thread" ]; } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + features = [ "server" ]; + } ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; @@ -5985,9 +5887,9 @@ rec { }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "0vbl4si1mny87pmqxxg6wday45pcc8bvpcrf46cpwwi4606qgy2n"; + sha256 = "1cq96c16hxsfvcd26ip1v3sg9952mi89snqdawc5whw14cjdlh4c"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6017,7 +5919,7 @@ rec { "base64" = [ "dep:base64" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/transport" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; "gen-tonic-messages" = [ "tonic" "prost" ]; "hex" = [ "dep:hex" ]; "internal-logs" = [ "tracing" ]; @@ -6038,15 +5940,10 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "0w4mycm070f4knvi1x5v199apd1fvi0712qiyv0pz70889havpw4"; + sha256 = "02r99lz30zrb8870vivww8cvwhdi78v5fv5sq6mr8wyls4hzppmg"; dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - optional = true; - } { name = "futures-channel"; packageId = "futures-channel"; @@ -6077,10 +5974,10 @@ rec { } { name = "rand"; - packageId = "rand 0.8.5"; + packageId = "rand 0.9.0"; optional = true; usesDefaultFeatures = false; - features = [ "std" "std_rng" "small_rng" ]; + features = [ "std" "std_rng" "small_rng" "os_rng" "thread_rng" ]; } { name = "serde_json"; @@ -6112,10 +6009,9 @@ rec { } ]; features = { - "async-std" = [ "dep:async-std" ]; - "async-trait" = [ "dep:async-trait" ]; "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" ]; + "experimental_logs_concurrent_log_processor" = [ "logs" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" ]; "experimental_trace_batch_span_processor_with_async_runtime" = [ "trace" ]; @@ -6124,25 +6020,24 @@ rec { "internal-logs" = [ "tracing" ]; "jaeger_remote_sampler" = [ "trace" "opentelemetry-http" "http" "serde" "serde_json" "url" ]; "logs" = [ "opentelemetry/logs" "serde_json" ]; - "metrics" = [ "opentelemetry/metrics" "glob" "async-trait" ]; + "metrics" = [ "opentelemetry/metrics" "glob" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "rand" = [ "dep:rand" ]; - "rt-async-std" = [ "async-std" "experimental_async_runtime" ]; "rt-tokio" = [ "tokio" "tokio-stream" "experimental_async_runtime" ]; "rt-tokio-current-thread" = [ "tokio" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-async-std" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "tracing" = [ "dep:tracing" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "async-trait" "default" "experimental_async_runtime" "glob" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "serde_json" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "glob" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "serde_json" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" "tracing" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -8796,13 +8691,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.92.0"; + version = "0.93.2"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; libName = "stackable_operator"; authors = [ @@ -8857,7 +8752,7 @@ rec { name = "k8s-openapi"; packageId = "k8s-openapi"; usesDefaultFeatures = false; - features = [ "schemars" "v1_32" ]; + features = [ "schemars" "v1_33" ]; } { name = "kube"; @@ -8960,8 +8855,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8995,8 +8890,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; libName = "stackable_shared"; authors = [ @@ -9036,8 +8931,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; libName = "stackable_telemetry"; authors = [ @@ -9046,7 +8941,8 @@ rec { dependencies = [ { name = "axum"; - packageId = "axum 0.8.3"; + packageId = "axum"; + features = [ "http2" ]; } { name = "clap"; @@ -9141,14 +9037,42 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; libName = "stackable_versioned"; authors = [ "Stackable GmbH " ]; dependencies = [ + { + name = "k8s-version"; + packageId = "k8s-version"; + optional = true; + features = [ "serde" ]; + } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + features = [ "url" ]; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + } + { + name = "serde_yaml"; + packageId = "serde_yaml"; + optional = true; + } { name = "stackable-versioned-macros"; packageId = "stackable-versioned-macros"; @@ -9156,7 +9080,7 @@ rec { ]; features = { "full" = [ "k8s" ]; - "k8s" = [ "stackable-versioned-macros/k8s" ]; + "k8s" = [ "stackable-versioned-macros/k8s" "dep:k8s-version" "dep:serde_json" "dep:serde_yaml" "dep:schemars" "dep:serde" ]; }; resolvedDefaultFeatures = [ "k8s" ]; }; @@ -9167,8 +9091,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "e0309b9f85413846984ff16f6e522d0f720dee43"; + sha256 = "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -9193,7 +9117,7 @@ rec { packageId = "k8s-openapi"; optional = true; usesDefaultFeatures = false; - features = [ "schemars" "v1_32" ]; + features = [ "schemars" "v1_33" ]; } { name = "k8s-version"; @@ -9220,14 +9144,6 @@ rec { packageId = "syn 2.0.100"; } ]; - devDependencies = [ - { - name = "k8s-openapi"; - packageId = "k8s-openapi"; - usesDefaultFeatures = false; - features = [ "schemars" "v1_32" ]; - } - ]; features = { "full" = [ "k8s" ]; "k8s" = [ "dep:kube" "dep:k8s-openapi" ]; @@ -9899,7 +9815,7 @@ rec { "time" = [ "tokio/time" ]; "tokio-util" = [ "dep:tokio-util" ]; }; - resolvedDefaultFeatures = [ "default" "net" "time" ]; + resolvedDefaultFeatures = [ "default" "time" ]; }; "tokio-util" = rec { crateName = "tokio-util"; @@ -10015,22 +9931,11 @@ rec { "Lucio Franco " ]; dependencies = [ - { - name = "async-stream"; - packageId = "async-stream"; - optional = true; - } { name = "async-trait"; packageId = "async-trait"; optional = true; } - { - name = "axum"; - packageId = "axum 0.7.9"; - optional = true; - usesDefaultFeatures = false; - } { name = "base64"; packageId = "base64 0.22.1"; @@ -10044,11 +9949,6 @@ rec { packageId = "flate2"; optional = true; } - { - name = "h2"; - packageId = "h2"; - optional = true; - } { name = "http"; packageId = "http"; @@ -10093,12 +9993,6 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } - { - name = "socket2"; - packageId = "socket2"; - optional = true; - features = [ "all" ]; - } { name = "tokio"; packageId = "tokio"; @@ -10156,7 +10050,7 @@ rec { "transport" = [ "server" "channel" ]; "zstd" = [ "dep:zstd" ]; }; - resolvedDefaultFeatures = [ "channel" "codegen" "gzip" "prost" "router" "server" "transport" ]; + resolvedDefaultFeatures = [ "channel" "codegen" "gzip" "prost" ]; }; "tower 0.4.13" = rec { crateName = "tower"; @@ -10696,9 +10590,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.29.0"; + version = "0.30.0"; edition = "2021"; - sha256 = "0dnca0b7bxbp6gd64skkvzy3p58yjh35kvnxpggz7sfwd4jjs7vj"; + sha256 = "0i64g7cyrdpzkc2zixz8bd0v1icha63ifcdwpvc3z0gmsr5pd3px"; libName = "tracing_opentelemetry"; dependencies = [ { diff --git a/Cargo.toml b/Cargo.toml index 61407770..64adb8ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/nifi-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.92.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.93.2" } anyhow = "1.0" built = { version = "0.7", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 9dcf61ef..3b97911f 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,10 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#k8s-version@0.1.2": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-operator-derive@0.3.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-operator@0.92.0": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-shared@0.0.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-telemetry@0.6.0": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-versioned-macros@0.7.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-versioned@0.7.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#k8s-version@0.1.3": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#stackable-operator-derive@0.3.1": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#stackable-operator@0.93.2": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#stackable-shared@0.0.1": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#stackable-telemetry@0.6.0": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#stackable-versioned-macros@0.7.1": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.2#stackable-versioned@0.7.1": "0604b0vvlzhh7a6pmcnm6k72pnkcv3f171n9981vj06wsbj9j98r", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index d01d7486..b3884ece 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -37,7 +37,7 @@ spec: items: properties: authenticationClass: - description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users. + description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users type: string oidc: description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. @@ -48,7 +48,7 @@ spec: type: string extraScopes: default: [] - description: An optional list of extra scopes which get merged with the scopes defined in the [`AuthenticationClass`]. + description: An optional list of extra scopes which get merged with the scopes defined in the `AuthenticationClass`. items: type: string type: array @@ -111,6 +111,66 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + customComponentsGitSync: + default: [] + description: The `customComponentsGitSync` setting allows configuring custom components to mount via `git-sync`. Learn more in the documentation for [Loading custom components](https://docs.stackable.tech/home/nightly/nifi/usage_guide/custom-components.html#git_sync). + items: + properties: + branch: + default: main + description: |- + The branch to clone; defaults to `main`. + + Since git-sync v4.x.x this field is mapped to the flag `--ref`. + type: string + credentialsSecret: + description: |- + The name of the Secret used to access the repository if it is not public. + + The referenced Secret must include two fields: `user` and `password`. The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the git-sync [documentation]. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string + depth: + default: 1 + description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. + format: uint32 + minimum: 0.0 + type: integer + gitFolder: + default: / + description: |- + Location in the Git repository containing the resource; defaults to the root folder. + + It can optionally start with `/`, however, no trailing slash is recommended. An empty string (``) or slash (`/`) corresponds to the root folder in Git. + type: string + gitSyncConf: + additionalProperties: + type: string + default: {} + description: |- + A map of optional configuration settings that are listed in the git-sync [documentation]. + + Also read the git-sync [example] in our documentation. These settings are not verified. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example + type: object + repo: + description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.' + format: uri + type: string + wait: + default: 20s + description: |- + The synchronization interval, e.g. `20s` or `5m`; defaults to `20s`. + + Since git-sync v4.x.x this field is mapped to the flag `--period`. + type: string + required: + - repo + type: object + type: array extraVolumes: description: Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for example make client certificates, keytabs or similar things available to processors. These volumes will be mounted into all pods at `/stackable/userdata/{volumename}`. See also the [external files usage guide](https://docs.stackable.tech/home/nightly/nifi/usage_guide/extra-volumes). items: diff --git a/docs/modules/nifi/pages/usage_guide/custom-components.adoc b/docs/modules/nifi/pages/usage_guide/custom-components.adoc new file mode 100644 index 00000000..210aa925 --- /dev/null +++ b/docs/modules/nifi/pages/usage_guide/custom-components.adoc @@ -0,0 +1,299 @@ +[#custom-components] += Loading custom components +:description: Load custom NiFi components for enhanced functionality. +:nifi-docs-developers-guide: https://nifi.apache.org/docs/nifi-docs/html/developer-guide.html +:nifi-docs-python-developers-guide: https://nifi.apache.org/nifi-docs/python-developer-guide.html +:nifi-docs-flowfile-source: https://nifi.apache.org/nifi-docs/python-developer-guide.html#flowfile-source +:git-sync-docs: https://github.com/kubernetes/git-sync/tree/v4.2.4#manual + +You can develop or use custom components for Apache NiFi, typically custom processors, to extend its functionality. + +There are currently two types of custom components: + +1. Custom NiFi Archives (NARs) + + The {nifi-docs-developers-guide}[NiFi Developer’s Guide] provides further information on how to develop custom NARs. +2. Starting with NiFi 2.0, also custom Python extensions can be used. + + In the {nifi-docs-python-developers-guide}[NiFi Python Developer’s Guide], the development of custom components using Python is described. + +The Stackable image contains the required tooling for both types. +For instance, a supported Python version is included. +NARs are only loaded once, but for Python scripts, hot-reloading is supported. + +TIP: You might need to refresh your browser window to see the new or changed Python components. + +Several options are described below, to add custom components to NiFi. + +[TIP] +==== +Check the logs of a NiFi node to ensure that the custom components are actually loaded, e.g.: + +[source,console] +---- +$ kubectl logs nifi-node-default-0 +… +… INFO [main] o.a.n.n.StandardExtensionDiscoveringManager Loaded extensions for tech.stackable.nifi:nifi-sample-nar:1.0.0 in 6 millis +… +… INFO [main] o.a.n.n.StandardExtensionDiscoveringManager Discovered Python Processor Greet +… +---- +==== + +[#git-sync] +== Synchronize with git-sync + +Custom NiFi components can be synchronized from a Git repository directly into the NiFi pods with git-sync. +The {crd-docs}/nifi.stackable.tech/nificluster/v1alpha1[NifiCluster CRD] allows the specification of one or multiple Git repositories: + +[source,yaml] +---- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +spec: + clusterConfig: + customComponentsGitSync: # <1> + - repo: https://example.com/git/custom-nifi-components # <2> + branch: main # <3> + gitFolder: path/to/the/components # <4> + depth: 10 # <5> + wait: 10s # <6> + credentialsSecret: git-credentials # <7> + gitSyncConf: # <8> + --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt + - repo: https://example.com/git/other-nifi-components # <9> + nodes: + config: + logging: + enableVectorAgent: true + containers: + git-sync: # <10> + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-credentials +type: Opaque +data: + user: ... + password: ... +---- +<1> If the optional field `customComponentsGitSync` is defined, then containers running git-sync are deployed to synchronize the specified repositories. +<2> The git repository URL that will be cloned +<3> The git revision (branch, tag, or hash) to check out; defaults to the `main` branch +<4> Location in the Git repository containing the NiFi components; defaults to the root folder +<5> The depth of synchronizing, i.e. the number of commits to clone; defaults to 1 +<6> The synchronization interval, e.g. `20s` or `5m`; defaults to `20s` +<7> The name of the Secret used to access the repository if it is not public. + + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the {git-sync-docs}[git-sync documentation]. +<8> A map of optional configuration settings that are listed in the {git-sync-docs}[git-sync documentation]. + + These settings are not verified. +<9> Multiple repositories can be defined. Only the `repo` field is mandatory. +<10> Logging can be configured as described in xref:concepts:logging.adoc[]. + As git-sync is a command-line tool, just its output is logged and no fine-grained log configuration is possible. + All git-sync containers are configured via the one `git-sync` field. + +It cannot be specified, if a repository contains NiFi Archives or Python components. +The operator just configures each repository for both types. +In particular, the parameters `nifi.nar.library.directory.+*+` and `nifi.python.extensions.source.directory.+*+` are set. + +[#config-map] +== Mount as a ConfigMap + +Custom components can also be stored in ConfigMaps. +This makes especially sense for Python components, but NAR files work as well. +This way, the components are stored and versioned alongside your NifiCluster itself. + +// Technically it's yaml, but most of the content is Python +[source,python] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi-processors +data: + CreateFlowFileProcessor.py: | + from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult + + class CreateFlowFile(FlowFileSource): + class Java: + implements = ['org.apache.nifi.python.processor.FlowFileSource'] + + class ProcessorDetails: + version = '0.0.1-SNAPSHOT' + description = '''A Python processor that creates FlowFiles.''' + + def __init__(self, **kwargs): + pass + + def create(self, context): + return FlowFileSourceResult( + relationship = 'success', + attributes = {'greeting': 'hello'}, + contents = 'Hello World!' + ) +binaryData: + custom-nifi-processor-1.0.0.nar: ... +---- + +The Python script is taken from {nifi-docs-flowfile-source}[the offical NiFi Python developer guide]. + +Afterwards, we need to mount the ConfigMap as described in xref:nifi:usage_guide/extra-volumes.adoc[] and extend the `nifi.properties` file: + +[source,yaml] +---- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: simple-nifi +spec: + image: + productVersion: 2.2.0 + clusterConfig: + authentication: + - authenticationClass: simple-nifi-admin-user + extraVolumes: # <1> + - name: nifi-processors + configMap: + name: nifi-processors + listenerClass: external-unstable + sensitiveProperties: + keySecret: nifi-sensitive-property-key + autoGenerate: true + zookeeperConfigMapName: simple-nifi-znode + nodes: + configOverrides: + nifi.properties: + nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ # <2> + nifi.python.extensions.source.directory.myCustomLibs: /stackable/userdata/nifi-processors/ + roleGroups: + default: + replicas: 1 +---- +<1> Specify your ConfigMaps here. +<2> The directory name after `userdata` has to match the name of the volume, while `myCustomLibs` is a name you can freely change. + +[#custom-image] +== Custom Docker image + +You can extend the official Stackable NiFi image by copying the required NAR files into NiFi's classpath or the Python files into the default extension directory. +The benefit of this method is that there is no need for any config overrides or extra mounts, you can use any of the NiFi examples, swap the image and your components will be available. +But this means, you will need to have access to a registry to push your custom image to. + +The basic Dockerfile below shows how to achieve this: + +[source,Dockerfile] +---- +FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable25.3.0 +COPY /path/to/your/nar.file /stackable/nifi/lib/ +COPY /path/to/your/Python.file /stackable/nifi/python/extensions/ +---- + +You then need to make this image available to your Kubernetes cluster and specify it in your NiFi resource as described in xref:concepts:product_image_selection.adoc[]. + +[source,yaml] +---- +spec: + image: + productVersion: 2.2.0 + custom: oci-registry.company.org/nifi:2.2.0-customprocessors +---- + +Also read the xref:guides:custom-images.adoc[Using customized product images] guide for additional information. + +[#pvc] +== Using the official image + +If you do not want to create a custom image or do not have access to an image registry, you can use the extra volume mount functionality to mount a volume containing your custom components and configure NiFi to read these from the mounted volumes. + +For this to work, you will need to prepare a PersistentVolumeClaim (PVC) containing your components. +Usually the best way to do this is to mount the PVC into a temporary container and then `kubectl cp` the NAR or Python files into that volume. + +The following listing shows an example of creating the PVC and the Pod: + +[source, yaml] +---- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nifi-processors +spec: + accessModes: + - ReadWriteOnce # <1> + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: processorcopy +spec: + volumes: + - name: nifi-processors + persistentVolumeClaim: + claimName: nifi-processors + containers: + - name: copycontainer + image: alpine + command: + - tail + - -f + - /dev/null + volumeMounts: + - mountPath: /volume + name: nifi-processors +---- +<1> Please note that this access mode means that you can only use this volume with a single NiFi Pod. + For a distributed scenario, an additional list value of `ReadOnlyMany` could be specified here if this is supported. + +The commands to then copy the NAR bundle and Python files into the PVC is: + +[source,bash] +---- +kubectl cp /path/to/component.nar processorcopy:/volume/ +kubectl cp /path/to/Python.files processorcopy:/volume/ +---- + +Now you can mount the extra volume into your NiFi instance as described in xref:nifi:usage_guide/extra-volumes.adoc[]. + +After this is done, your components are available within the NiFi Pods and NiFi can be configured to load components from the volume: + +[source,yaml] +---- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: simple-nifi +spec: + image: + productVersion: 2.2.0 + clusterConfig: + authentication: + - authenticationClass: simple-nifi-admin-user + extraVolumes: # <1> + - name: nifi-processors + persistentVolumeClaim: + claimName: nifi-processors + listenerClass: external-unstable + sensitiveProperties: + keySecret: nifi-sensitive-property-key + autoGenerate: true + zookeeperConfigMapName: simple-nifi-znode + nodes: + configOverrides: + nifi.properties: + nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ # <2> + nifi.python.extensions.source.directory.myCustomLibs: /stackable/userdata/nifi-processors/ + roleGroups: + default: + replicas: 1 +---- +<1> Specify your prepared PVC here. +<2> The directory name after `userdata` has to match the name of the volume, while `myCustomLibs` is a name you can freely change. diff --git a/docs/modules/nifi/pages/usage_guide/custom-components/custom-nars.adoc b/docs/modules/nifi/pages/usage_guide/custom-components/custom-nars.adoc deleted file mode 100644 index 78d5da2a..00000000 --- a/docs/modules/nifi/pages/usage_guide/custom-components/custom-nars.adoc +++ /dev/null @@ -1,129 +0,0 @@ -= Custom `.nar` files - -:description: Load custom NiFi components by using custom Docker images or mounting external volumes with nar files for enhanced functionality. -:nifi-docs-custom-components: https://nifi.apache.org/docs/nifi-docs/html/developer-guide.html#introduction - -You can develop {nifi-docs-custom-components}[custom components] for Apache NiFi, typically custom processors, to extend its functionality. -For these to be available in the NiFi UI, they need to be present in the classpath at startup, so the nar bundles have to be injected into your NiFi instance. - -The Stackable Data Platform supports two ways to achive this goal, both of which are described below. - -Please note that building your own Docker image is the recommended solution, as giving multiple pods access to a shared PVC can often be tricky (see comment on access mode in the second section). - -== Custom Docker image - -You can extend the official Stackable NiFi image by copying the required _nar_ files into NiFi's classpath. -The benefit of this method is that there is no need for any config overrides or extra mounts, you can use any of the NiFi examples, swap the image and your components will be available. -But this means you will need to have access to a registry to push your custom image to. - -The basic Dockerfile below shows how to achieve this: - -[source,Dockerfile] ----- -FROM oci.stackable.tech/sdp/nifi:1.27.0-stackable0.0.0-dev -COPY /path/to/your/nar.file /stackable/nifi/lib/ ----- - -You then need to make this image available to your Kubernetes cluster and specify it in your NiFi resource as described in xref:concepts:product_image_selection.adoc[]. - -[source,yaml] ----- -spec: - image: - productVersion: 1.27.0 - custom: "docker.company.org/nifi:1.27.0-customprocessor" ----- - -Also read the xref:guides:custom-images.adoc[Using customized product images] guide for additional information. - -== Using the official image - -If you don't want to create a custom image or don't have access to an image registry, you can use the extra volume mount functionality to mount a volume containing your custom components and configure NiFi to read these from the mounted volumes. - -For this to work you'll need to prepare a PersistentVolumeClaim (PVC) containing your components. -Usually the best way to do this is to mount the PVC into a temporary container and then `kubectl cp` the _nar_ files into that volume. - -The following listing shows an example of creating the PVC and the Pod: - -[source, yaml] ----- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nifi-processor -spec: - accessModes: - - ReadWriteOnce # <1> - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: Pod -metadata: - name: processorcopy -spec: - volumes: - - name: nifi-processor - persistentVolumeClaim: - claimName: nifi-processor - containers: - - name: copycontainer - image: alpine - command: - - tail - - "-f" - - "/dev/null" - volumeMounts: - - mountPath: "/volume" - name: nifi-processor ----- - -<1> Please note that this access mode means that you can only use this volume with a single NiFi Pod. - For a distributed scenario, an additional list value of `ReadOnlyMany` could be specified here if this is supported. - -The command to then copy the nar bundle into the PVC is: - -[source,bash] ----- -kubectl cp /path/to/component.nar processorcopy:/volume/ ----- - -Now you can mount the extra volume into your NiFi instance as described in xref:nifi:usage_guide/extra-volumes.adoc[]. - -After this is done, your components are available within the NiFi Pods and NiFi can be configured to load these at startup by adding an extra directory to the classpath: - - -[source,yaml] ----- -apiVersion: nifi.stackable.tech/v1alpha1 -kind: NifiCluster -metadata: - name: simple-nifi -spec: - image: - productVersion: 1.27.0 - clusterConfig: - authentication: - - authenticationClass: simple-nifi-admin-user - extraVolumes: # <1> - - name: nifi-processor - persistentVolumeClaim: - claimName: nifi-processor - listenerClass: external-unstable - sensitiveProperties: - keySecret: nifi-sensitive-property-key - autoGenerate: true - zookeeperConfigMapName: simple-nifi-znode - nodes: - config: - configOverrides: - nifi.properties: - nifi.nar.library.directory.myCustomLibs: "../userdata/nifi-processor/" # <2> - roleGroups: - default: - replicas: 1 ----- - -<1> Specify your prepared PVC here. -<2> The directory name after `userdata` has to match the name of the volume, while `myCustomLibs` is a name you can freely change. diff --git a/docs/modules/nifi/pages/usage_guide/custom-components/custom-python-processors.adoc b/docs/modules/nifi/pages/usage_guide/custom-components/custom-python-processors.adoc deleted file mode 100644 index 99cffcc6..00000000 --- a/docs/modules/nifi/pages/usage_guide/custom-components/custom-python-processors.adoc +++ /dev/null @@ -1,165 +0,0 @@ -= Custom Python processors - -In NiFi 2.0, support for custom processors written in Python was added. -The Stackable images already contain the required tooling, such as - obviously - a supported Python version. - -== General configuration - -[source,yaml] ----- -spec: - nodes: - configOverrides: - nifi.properties: - # The command used to launch Python. - # This property must be set to enable Python-based processors. - nifi.python.command: python3 - # The directory that NiFi should look in to find custom Python-based - # Processors. - nifi.python.extensions.source.directory.custom: /nifi-python-extensions - # The directory that contains the Python framework for communicating - # between the Python and Java processes. - nifi.python.framework.source.directory: /stackable/nifi/python/framework/ - # The working directory where NiFi should store artifacts - # This property defaults to ./work/python but if you want to mount an - # emptyDir for the working directory then another directory has to be - # set to avoid ownership conflicts with ./work/nar. - nifi.python.working.directory: /nifi-python-working-directory ----- - -== Getting Python scripts into NiFi - -TIP: NiFi should hot-reload the Python scripts. You might need to refresh your browser window to see the new processor. - -[#configmap] -=== 1. Mount as ConfigMap - -The easiest way is defining a ConfigMap and mounting it as follows. -This way, the Python processors are stored and versioned alongside your NifiCluster itself. - -// Technically it's yaml, but the most content is Python -[source,python] ----- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nifi-python-extensions -data: - CreateFlowFileProcessor.py: | - from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult - - class CreateFlowFile(FlowFileSource): - class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] - - class ProcessorDetails: - version = '0.0.1-SNAPSHOT' - description = '''A Python processor that creates FlowFiles.''' - - def __init__(self, **kwargs): - pass - - def create(self, context): - return FlowFileSourceResult( - relationship = 'success', - attributes = {'greeting': 'hello'}, - contents = 'Hello World!' - ) ----- - -The Python script is taken from https://nifi.apache.org/nifi-docs/python-developer-guide.html#flowfile-source[the offical NiFi Python developer guide]. - -You can add multiple Python scripts in the ConfigMap. -Afterwards we need to mount the Python scripts into `/nifi-python-extensions`: - -[source,yaml] ----- -spec: - nodes: - podOverrides: - spec: - containers: - - name: nifi - volumeMounts: - - name: nifi-python-extensions - mountPath: /nifi-python-extensions - - name: nifi-python-working-directory - mountPath: /nifi-python-working-directory - volumes: - - name: nifi-python-extensions - configMap: - name: nifi-python-extensions - - name: nifi-python-working-directory - emptyDir: {} ----- - -[#git-sync] -=== 2. Use git-sync - -As an alternative you can use `git-sync` to keep your Python processors up to date. -You need to add a sidecar using podOverrides that syncs into a shared volume between the `nifi` and `git-sync` container. - -The following snippet can serve as a starting point (the Git repo has the folder `processors` with the Python scripts inside). - -[source,yaml] ----- -spec: - nodes: - podOverrides: - spec: - containers: - - name: nifi - volumeMounts: - - name: nifi-python-extensions - mountPath: /nifi-python-extensions - - name: nifi-python-working-directory - mountPath: /nifi-python-working-directory - - name: git-sync - image: registry.k8s.io/git-sync/git-sync:v4.2.3 - args: - - --repo=https://github.com/stackabletech/nifi-talk - - --root=/nifi-python-extensions - - --period=10s - volumeMounts: - - name: nifi-python-extensions - mountPath: /nifi-python-extensions - volumes: - - name: nifi-python-extensions - emptyDir: {} - - name: nifi-python-working-directory - emptyDir: {} ----- - -Afterwards you need to update your source directory (the one you added previously) accordingly, to point into the Git subfolder you have. - -[source,yaml] ----- -spec: - nodes: - configOverrides: - nifi.properties: - # Replace the property from the previous step - # Format is /nifi-python-extensions/// - nifi.python.extensions.source.directory.custom: > - /nifi-python-extensions/nifi-talk/processors/ ----- - -=== 3. Use PersistentVolume - -You can also mount a PVC below `/nifi-python-extensions` using podOverrides and shell into the NiFi Pod to make changes. -However, the <> or <> approach is recommended. - -== Check processors have been loaded - -NiFi logs every Python processor it found. -You can use that to check if the processors have been loaded. - -[source,console] ----- -$ kubectl logs nifi-2-0-0-node-default-0 -c nifi \ - | grep 'Discovered.*Python Processor' -… INFO [main] … Discovered Python Processor PythonZgrepProcessor -… INFO [main] … Discovered Python Processor TransformOpenskyStates -… INFO [main] … Discovered Python Processor UpdateAttributeFileLookup -… INFO [main] … Discovered or updated 3 Python Processors in 64 millis ----- diff --git a/docs/modules/nifi/pages/usage_guide/custom-components/index.adoc b/docs/modules/nifi/pages/usage_guide/custom-components/index.adoc deleted file mode 100644 index 710b8812..00000000 --- a/docs/modules/nifi/pages/usage_guide/custom-components/index.adoc +++ /dev/null @@ -1,9 +0,0 @@ -= Loading custom components -:description: Load custom NiFi components for enhanced functionality. - -You can develop or use custom components for Apache NiFi, typically custom processors, to extend its functionality. - -There are currently two types of custom components: - -1. xref:nifi:usage_guide/custom-components/custom-nars.adoc[] -2. Starting with NiFi 2.0 you can also use xref:nifi:usage_guide/custom-components/custom-python-processors.adoc[] diff --git a/docs/modules/nifi/partials/nav.adoc b/docs/modules/nifi/partials/nav.adoc index b753ce31..a3114057 100644 --- a/docs/modules/nifi/partials/nav.adoc +++ b/docs/modules/nifi/partials/nav.adoc @@ -16,9 +16,7 @@ ** xref:nifi:usage_guide/exposing-processors/index.adoc[] *** xref:nifi:usage_guide/exposing-processors/http.adoc[] *** xref:nifi:usage_guide/exposing-processors/tcp.adoc[] -** xref:nifi:usage_guide/custom-components/index.adoc[] -*** xref:nifi:usage_guide/custom-components/custom-nars.adoc[] -*** xref:nifi:usage_guide/custom-components/custom-python-processors.adoc[] +** xref:nifi:usage_guide/custom-components.adoc[] ** xref:nifi:usage_guide/operations/index.adoc[] *** xref:nifi:usage_guide/operations/cluster-operations.adoc[] *** xref:nifi:usage_guide/operations/pod-placement.adoc[] diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index f01596b5..7120c9b7 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -8,6 +8,7 @@ use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{ResultExt, Snafu, ensure}; use stackable_operator::{ commons::resources::Resources, + crd::git_sync, memory::MemoryQuantity, product_config_utils::{ ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, @@ -34,6 +35,7 @@ use crate::{ pub mod jvm; pub const NIFI_CONFIG_DIRECTORY: &str = "/stackable/nifi/conf"; +pub const NIFI_PYTHON_WORKING_DIRECTORY: &str = "/nifi-python-working-directory"; pub const NIFI_BOOTSTRAP_CONF: &str = "bootstrap.conf"; pub const NIFI_PROPERTIES: &str = "nifi.properties"; @@ -150,6 +152,7 @@ pub fn build_nifi_properties( auth_config: &NifiAuthenticationConfig, overrides: BTreeMap, product_version: &str, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, ) -> Result { // TODO: Remove once we dropped support for all NiFi 1.x versions let is_nifi_1 = product_version.starts_with("1."); @@ -623,6 +626,55 @@ pub fn build_nifi_properties( } } + //#################### + // Custom components # + //#################### + // NiFi 1.x does not support Python components and the Python configuration below is just + // ignored. + + // The command used to launch Python. + // This property must be set to enable Python-based processors. + properties.insert("nifi.python.command".to_string(), "python3".to_string()); + + // The directory that contains the Python framework for communicating between the Python and + // Java processes. + properties.insert( + "nifi.python.framework.source.directory".to_string(), + "/stackable/nifi/python/framework/".to_string(), + ); + + // The working directory where NiFi should store artifacts; + // This property defaults to ./work/python but if you want to mount an emptyDir for the working + // directory then another directory has to be set to avoid ownership conflicts with ./work/nar. + properties.insert( + "nifi.python.working.directory".to_string(), + NIFI_PYTHON_WORKING_DIRECTORY.to_string(), + ); + + // The default directory that NiFi should look in to find custom Python-based components. + // This directory is mentioned in the documentation + // (docs/modules/nifi/pages/usage_guide/custom-components.adoc), so do not change it! + properties.insert( + "nifi.python.extensions.source.directory.default".to_string(), + "/stackable/nifi/python/extensions/".to_string(), + ); + + for (i, git_folder) in git_sync_resources + .git_content_folders_as_string() + .into_iter() + .enumerate() + { + // The directory that NiFi should look in to find custom Python-based components. + properties.insert( + format!("nifi.python.extensions.source.directory.{i}"), + git_folder.clone(), + ); + + // The directory that NiFi should look in to find custom Java-based components. + properties.insert(format!("nifi.nar.library.directory.{i}"), git_folder); + } + //########################## + // override with config overrides properties.extend(overrides); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 3e52094e..993647fd 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -26,11 +26,9 @@ use stackable_operator::{ }, client::Client, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{ - authentication::oidc::AuthenticationProvider, - product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, - }, + commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, config::fragment, + crd::{authentication::oidc, git_sync}, k8s_openapi::{ DeepMerge, api::{ @@ -52,6 +50,7 @@ use stackable_operator::{ kvp::{Label, Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, + product_config_utils::env_vars_from_rolegroup_config, product_logging::{ self, framework::{ @@ -77,8 +76,9 @@ use crate::{ OPERATOR_NAME, config::{ self, JVM_SECURITY_PROPERTIES_FILE, NIFI_BOOTSTRAP_CONF, NIFI_CONFIG_DIRECTORY, - NIFI_PROPERTIES, NIFI_STATE_MANAGEMENT_XML, NifiRepository, build_bootstrap_conf, - build_nifi_properties, build_state_management_xml, validated_product_config, + NIFI_PROPERTIES, NIFI_PYTHON_WORKING_DIRECTORY, NIFI_STATE_MANAGEMENT_XML, NifiRepository, + build_bootstrap_conf, build_nifi_properties, build_state_management_xml, + validated_product_config, }, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, CurrentlySupportedListenerClasses, @@ -109,6 +109,7 @@ pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); const DOCKER_IMAGE_BASE_NAME: &str = "nifi"; +const LOG_VOLUME_NAME: &str = "log"; pub struct Ctx { pub client: Client, @@ -258,6 +259,9 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, + #[snafu(display("invalid git-sync specification"))] + InvalidGitSyncSpec { source: git_sync::v1alpha1::Error }, + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, @@ -509,6 +513,16 @@ pub async fn reconcile_nifi( .merged_config(&NifiRole::Node, rolegroup_name) .context(FailedToResolveConfigSnafu)?; + let git_sync_resources = git_sync::v1alpha1::GitSyncResources::new( + &nifi.spec.cluster_config.custom_components_git_sync, + &resolved_product_image, + &env_vars_from_rolegroup_config(rolegroup_config), + &[], + LOG_VOLUME_NAME, + &merged_config.logging.for_container(&Container::GitSync), + ) + .context(InvalidGitSyncSpecSnafu)?; + let rg_service = build_node_rolegroup_service(nifi, &resolved_product_image, &rolegroup)?; @@ -531,6 +545,7 @@ pub async fn reconcile_nifi( rolegroup_config, &merged_config, &proxy_hosts, + &git_sync_resources, ) .await?; @@ -555,6 +570,7 @@ pub async fn reconcile_nifi( rolling_upgrade_supported, replicas, &rbac_sa.name_any(), + &git_sync_resources, ) .await?; @@ -716,6 +732,7 @@ async fn build_node_rolegroup_config_map( rolegroup_config: &HashMap>, merged_config: &NifiConfig, proxy_hosts: &str, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, ) -> Result { tracing::debug!("building rolegroup configmaps"); @@ -784,6 +801,7 @@ async fn build_node_rolegroup_config_map( })? .clone(), resolved_product_image.product_version.as_ref(), + git_sync_resources, ) .with_context(|_| BuildProductConfigSnafu { rolegroup: rolegroup.clone(), @@ -893,6 +911,7 @@ async fn build_node_rolegroup_statefulset( rolling_update_supported: bool, replicas: Option, service_account_name: &str, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, ) -> Result { tracing::debug!("Building statefulset"); let role_group = role.role_groups.get(&rolegroup_ref.role_group); @@ -960,9 +979,11 @@ async fn build_node_rolegroup_statefulset( } if let NifiAuthenticationConfig::Oidc { oidc, .. } = authentication_config { - env_vars.extend(AuthenticationProvider::client_credentials_env_var_mounts( - oidc.client_credentials_secret_ref.clone(), - )); + env_vars.extend( + oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts( + oidc.client_credentials_secret_ref.clone(), + ), + ); } env_vars.extend(authorization_config.get_env_vars()); @@ -1083,7 +1104,7 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount("sensitiveproperty", "/stackable/sensitiveproperty") .context(AddVolumeMountSnafu)? - .add_volume_mount("log", STACKABLE_LOG_DIR) + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) .context(AddVolumeMountSnafu)? @@ -1159,7 +1180,7 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR) .context(AddVolumeMountSnafu)? - .add_volume_mount("log", STACKABLE_LOG_DIR) + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) .context(AddVolumeMountSnafu)? @@ -1212,9 +1233,31 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)?; } + let volume_name = "nifi-python-working-directory".to_string(); + pod_builder + .add_empty_dir_volume(&volume_name, None) + .context(AddVolumeSnafu)?; + container_nifi + .add_volume_mount(&volume_name, NIFI_PYTHON_WORKING_DIRECTORY) + .context(AddVolumeMountSnafu)?; + + container_nifi + .add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) + .context(AddVolumeMountSnafu)?; + // We want to add nifi container first for easier defaulting into this container pod_builder.add_container(container_nifi.build()); + for container in git_sync_resources.git_sync_containers.iter().cloned() { + pod_builder.add_container(container); + } + for container in git_sync_resources.git_sync_init_containers.iter().cloned() { + pod_builder.add_init_container(container); + } + pod_builder + .add_volumes(git_sync_resources.git_content_volumes.to_owned()) + .context(AddVolumeSnafu)?; + if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { @@ -1252,7 +1295,7 @@ async fn build_node_rolegroup_statefulset( product_logging::framework::vector_container( resolved_product_image, "config", - "log", + LOG_VOLUME_NAME, merged_config.logging.containers.get(&Container::Vector), ResourceRequirementsBuilder::new() .with_cpu_request("250m") @@ -1318,7 +1361,7 @@ async fn build_node_rolegroup_statefulset( }) .context(AddVolumeSnafu)? .add_empty_dir_volume( - "log", + LOG_VOLUME_NAME, // Set volume size to higher than theoretically necessary to avoid running out of disk space as log rotation triggers are only checked by Logback every 5s. Some( MemoryQuantity { @@ -1413,7 +1456,7 @@ async fn build_node_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: rolegroup_ref.object_name(), + service_name: Some(rolegroup_ref.object_name()), template: pod_template, update_strategy: Some(StatefulSetUpdateStrategy { type_: if rolling_update_supported { diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index f3e0b32e..561e5cfc 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -3,10 +3,7 @@ use std::future::Future; use snafu::{ResultExt, Snafu}; use stackable_operator::{ client::Client, - commons::authentication::{ - AuthenticationClass, AuthenticationClassProvider, ClientAuthenticationDetails, ldap, oidc, - static_, - }, + crd::authentication::{core as auth_core, ldap, oidc, r#static}, kube::{ResourceExt, runtime::reflector::ObjectRef}, }; @@ -34,19 +31,19 @@ pub enum Error { ))] AuthenticationClassProviderNotSupported { authentication_class_provider: String, - authentication_class: ObjectRef, + authentication_class: ObjectRef, }, #[snafu(display( "Nifi doesn't support skipping the LDAP TLS verification of the AuthenticationClass {authentication_class}" ))] NoLdapTlsVerificationNotSupported { - authentication_class: ObjectRef, + authentication_class: ObjectRef, }, #[snafu(display("invalid OIDC configuration"))] OidcConfigurationInvalid { - source: stackable_operator::commons::authentication::Error, + source: stackable_operator::crd::authentication::core::v1alpha1::Error, }, } @@ -55,14 +52,14 @@ type Result = std::result::Result; #[derive(Clone)] pub enum AuthenticationClassResolved { Static { - provider: static_::AuthenticationProvider, + provider: r#static::v1alpha1::AuthenticationProvider, }, Ldap { - provider: ldap::AuthenticationProvider, + provider: ldap::v1alpha1::AuthenticationProvider, }, Oidc { - provider: oidc::AuthenticationProvider, - oidc: oidc::ClientAuthenticationOptions<()>, + provider: oidc::v1alpha1::AuthenticationProvider, + oidc: oidc::v1alpha1::ClientAuthenticationOptions<()>, nifi: v1alpha1::NifiCluster, }, } @@ -72,19 +69,25 @@ impl AuthenticationClassResolved { nifi: &v1alpha1::NifiCluster, client: &Client, ) -> Result> { - let resolve_auth_class = |auth_details: ClientAuthenticationDetails| async move { - auth_details.resolve_class(client).await - }; + let resolve_auth_class = + |auth_details: auth_core::v1alpha1::ClientAuthenticationDetails| async move { + auth_details.resolve_class(client).await + }; AuthenticationClassResolved::resolve(nifi, resolve_auth_class).await } /// Retrieve all provided `AuthenticationClass` references. pub async fn resolve( nifi: &v1alpha1::NifiCluster, - resolve_auth_class: impl Fn(ClientAuthenticationDetails) -> R, + resolve_auth_class: impl Fn(auth_core::v1alpha1::ClientAuthenticationDetails) -> R, ) -> Result> where - R: Future>, + R: Future< + Output = Result< + auth_core::v1alpha1::AuthenticationClass, + stackable_operator::client::Error, + >, + >, { let mut resolved_auth_classes = vec![]; let auth_details = &nifi.spec.cluster_config.authentication; @@ -103,16 +106,18 @@ impl AuthenticationClassResolved { let auth_class_name = auth_class.name_any(); match &auth_class.spec.provider { - AuthenticationClassProvider::Static(provider) => { + auth_core::v1alpha1::AuthenticationClassProvider::Static(provider) => { resolved_auth_classes.push(AuthenticationClassResolved::Static { provider: provider.to_owned(), }) } - AuthenticationClassProvider::Ldap(provider) => { + auth_core::v1alpha1::AuthenticationClassProvider::Ldap(provider) => { if provider.tls.uses_tls() && !provider.tls.uses_tls_verification() { NoLdapTlsVerificationNotSupportedSnafu { - authentication_class: ObjectRef::::new( - &auth_class_name, + authentication_class: ObjectRef::< + auth_core::v1alpha1::AuthenticationClass, + >::new( + &auth_class_name ), } .fail()? @@ -121,7 +126,7 @@ impl AuthenticationClassResolved { provider: provider.to_owned(), }) } - AuthenticationClassProvider::Oidc(provider) => { + auth_core::v1alpha1::AuthenticationClassProvider::Oidc(provider) => { resolved_auth_classes.push(Ok(AuthenticationClassResolved::Oidc { provider: provider.to_owned(), oidc: entry @@ -131,11 +136,16 @@ impl AuthenticationClassResolved { nifi: nifi.clone(), })?) } - _ => AuthenticationClassProviderNotSupportedSnafu { - authentication_class_provider: auth_class.spec.provider.to_string(), - authentication_class: ObjectRef::::new(&auth_class_name), + _ => { + AuthenticationClassProviderNotSupportedSnafu { + authentication_class_provider: auth_class.spec.provider.to_string(), + authentication_class: + ObjectRef::::new( + &auth_class_name, + ), + } + .fail()? } - .fail()?, }; } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1eda9314..430fd8d7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -12,7 +12,6 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::{ affinity::StackableAffinity, - authentication::ClientAuthenticationDetails, cache::UserInformationCache, cluster_operation::ClusterOperation, opa::OpaConfig, @@ -26,6 +25,7 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, + crd::{authentication::core as auth_core, git_sync}, k8s_openapi::{ api::core::v1::{PodTemplateSpec, Volume}, apimachinery::pkg::api::resource::Quantity, @@ -121,7 +121,7 @@ pub mod versioned { /// Authentication options for NiFi (required). /// Read more about authentication in the [security documentation](DOCS_BASE_URL_PLACEHOLDER/nifi/usage_guide/security#authentication). // We don't add `#[serde(default)]` here, as we require authentication - pub authentication: Vec, + pub authentication: Vec, /// Authorization options. /// Learn more in the [NiFi authorization usage guide](DOCS_BASE_URL_PLACEHOLDER/nifi/usage-guide/security#authorization). @@ -150,6 +150,12 @@ pub mod versioned { #[serde(flatten)] pub clustering_backend: NifiClusteringBackend, + /// The `customComponentsGitSync` setting allows configuring custom components to mount via `git-sync`. + /// Learn more in the documentation for + /// [Loading custom components](DOCS_BASE_URL_PLACEHOLDER/nifi/usage_guide/custom-components.html#git_sync). + #[serde(default)] + pub custom_components_git_sync: Vec, + /// Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for /// example make client certificates, keytabs or similar things available to processors. These volumes will be /// mounted into all pods at `/stackable/userdata/{volumename}`. @@ -443,6 +449,7 @@ pub enum Container { Prepare, Vector, Nifi, + GitSync, } #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 4cfee3df..e0ff5b63 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -6,7 +6,7 @@ use futures::stream::StreamExt; use stackable_operator::{ YamlSchema, cli::{Command, ProductOperatorRun}, - commons::authentication::AuthenticationClass, + crd::authentication::core as auth_core, k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service}, @@ -120,7 +120,8 @@ async fn main() -> anyhow::Result<()> { ) .shutdown_on_signal() .watches( - client.get_api::>(&()), + client + .get_api::>(&()), watcher::Config::default(), move |_| { authentication_class_store diff --git a/rust/operator-binary/src/security/authentication.rs b/rust/operator-binary/src/security/authentication.rs index 24455634..4f6633c0 100644 --- a/rust/operator-binary/src/security/authentication.rs +++ b/rust/operator-binary/src/security/authentication.rs @@ -5,11 +5,7 @@ use stackable_operator::{ self, pod::{PodBuilder, container::ContainerBuilder}, }, - commons::authentication::{ - ldap, - oidc::{self, ClientAuthenticationOptions}, - static_, - }, + crd::authentication::{ldap, oidc, r#static}, k8s_openapi::api::core::v1::{KeyToPath, SecretVolumeSource, Volume}, }; @@ -42,7 +38,7 @@ pub enum Error { #[snafu(display("Failed to add LDAP volumes and volumeMounts to the Pod and containers"))] AddLdapVolumes { - source: stackable_operator::commons::authentication::ldap::Error, + source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, }, #[snafu(display("Failed to add OIDC volumes and volumeMounts to the Pod and containers"))] @@ -67,14 +63,14 @@ pub enum Error { #[allow(clippy::large_enum_variant)] pub enum NifiAuthenticationConfig { SingleUser { - provider: static_::AuthenticationProvider, + provider: r#static::v1alpha1::AuthenticationProvider, }, Ldap { - provider: ldap::AuthenticationProvider, + provider: ldap::v1alpha1::AuthenticationProvider, }, Oidc { - provider: oidc::AuthenticationProvider, - oidc: ClientAuthenticationOptions, + provider: oidc::v1alpha1::AuthenticationProvider, + oidc: oidc::v1alpha1::ClientAuthenticationOptions, nifi: v1alpha1::NifiCluster, }, } @@ -263,7 +259,9 @@ impl NifiAuthenticationConfig { } } -fn get_ldap_login_identity_provider(ldap: &ldap::AuthenticationProvider) -> Result { +fn get_ldap_login_identity_provider( + ldap: &ldap::v1alpha1::AuthenticationProvider, +) -> Result { let mut search_filter = ldap.search_filter.clone(); // If no search_filter is specified we will set a default filter that just searches for the user logging in using the specified uid field name diff --git a/rust/operator-binary/src/security/authorization.rs b/rust/operator-binary/src/security/authorization.rs index 51fdc717..8f0575c7 100644 --- a/rust/operator-binary/src/security/authorization.rs +++ b/rust/operator-binary/src/security/authorization.rs @@ -1,7 +1,7 @@ use indoc::{formatdoc, indoc}; use snafu::{OptionExt, Snafu}; use stackable_operator::{ - commons::authentication::ldap, + crd::authentication::ldap, k8s_openapi::api::core::v1::{ConfigMapKeySelector, EnvVar, EnvVarSource}, }; @@ -91,7 +91,7 @@ impl NifiAuthorizationConfig { fn get_default_ldap_authorizer( &self, - ldap: &ldap::AuthenticationProvider, + ldap: &ldap::v1alpha1::AuthenticationProvider, ) -> Result { let (username_file, _) = ldap .bind_credentials_mount_paths() diff --git a/rust/operator-binary/src/security/oidc.rs b/rust/operator-binary/src/security/oidc.rs index e8232be6..038fb28b 100644 --- a/rust/operator-binary/src/security/oidc.rs +++ b/rust/operator-binary/src/security/oidc.rs @@ -5,10 +5,8 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, client::Client, - commons::{ - authentication::oidc::{self, AuthenticationProvider, ClientAuthenticationOptions}, - tls_verification::{CaCert, TlsServerVerification, TlsVerification}, - }, + commons::tls_verification::{CaCert, TlsServerVerification, TlsVerification}, + crd::authentication::oidc, k8s_openapi::api::core::v1::Secret, kube::{ResourceExt, runtime::reflector::ObjectRef}, }; @@ -34,7 +32,7 @@ pub enum Error { #[snafu(display("invalid well-known OIDC configuration URL"))] InvalidWellKnownConfigUrl { - source: stackable_operator::commons::authentication::oidc::Error, + source: stackable_operator::crd::authentication::oidc::v1alpha1::Error, }, #[snafu(display("Nifi doesn't support skipping the OIDC TLS verification"))] @@ -105,8 +103,8 @@ pub fn build_oidc_admin_password_secret_name(nifi: &v1alpha1::NifiCluster) -> St /// Adds all the required configuration properties to enable OIDC authentication. pub fn add_oidc_config_to_properties( - provider: &oidc::AuthenticationProvider, - client_auth_options: &ClientAuthenticationOptions, + provider: &oidc::v1alpha1::AuthenticationProvider, + client_auth_options: &oidc::v1alpha1::ClientAuthenticationOptions, properties: &mut BTreeMap, ) -> Result<(), Error> { let well_known_url = provider @@ -118,7 +116,7 @@ pub fn add_oidc_config_to_properties( well_known_url.to_string(), ); let (oidc_client_id_env, oidc_client_secret_env) = - AuthenticationProvider::client_credentials_env_names( + oidc::v1alpha1::AuthenticationProvider::client_credentials_env_names( &client_auth_options.client_credentials_secret_ref, ); properties.insert( @@ -171,7 +169,7 @@ mod tests { #[case("/realms/sdp/////")] fn test_add_oidc_config(#[case] root_path: String) { let mut properties = BTreeMap::new(); - let provider = oidc::AuthenticationProvider::new( + let provider = oidc::v1alpha1::AuthenticationProvider::new( "keycloak.mycorp.org".to_owned().try_into().unwrap(), Some(443), root_path, @@ -186,7 +184,7 @@ mod tests { vec!["openid".to_owned()], None, ); - let oidc = ClientAuthenticationOptions { + let oidc = oidc::v1alpha1::ClientAuthenticationOptions { client_credentials_secret_ref: "nifi-keycloak-client".to_owned(), extra_scopes: vec![], product_specific_fields: (), diff --git a/tests/templates/kuttl/custom-components-git-sync/00-patch-ns.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/custom-components-git-sync/10-assert.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/10-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/10-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/custom-components-git-sync/10-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/10-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/10-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml b/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml new file mode 100644 index 00000000..e0766c49 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-zk-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/custom-components-git-sync/20-install-zk.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/20-install-zk.yaml.j2 new file mode 100644 index 00000000..ab2a9536 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/20-install-zk.yaml.j2 @@ -0,0 +1,28 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: test-zk +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: test-nifi-znode +spec: + clusterRef: + name: test-zk diff --git a/tests/templates/kuttl/custom-components-git-sync/30-assert.yaml b/tests/templates/kuttl/custom-components-git-sync/30-assert.yaml new file mode 100644 index 00000000..5735208f --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/30-assert.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-nifi-node-default +spec: + template: + spec: + terminationGracePeriodSeconds: 300 +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 new file mode 100644 index 00000000..a6153bd6 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 @@ -0,0 +1,845 @@ +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-user-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-user-credentials +stringData: + admin: admin +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: test-nifi +spec: + image: +{% if test_scenario['values']['nifi'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['nifi'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + listenerClass: external-unstable + zookeeperConfigMapName: test-nifi-znode + authentication: + - authenticationClass: nifi-users + sensitiveProperties: + keySecret: nifi-sensitive-property-key + autoGenerate: true + customComponentsGitSync: + - repo: https://github.com/stackabletech/nifi-operator + branch: ef61c87311ad2f57484c33245c9ed50908a1c785 + gitFolder: tests/templates/kuttl/custom-components-git-sync/java-processors + - repo: https://github.com/stackabletech/nifi-operator + branch: ef61c87311ad2f57484c33245c9ed50908a1c785 + gitFolder: tests/templates/kuttl/custom-components-git-sync/python-processors +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + podOverrides: + spec: + initContainers: + - name: init-flow + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - /bin/bash + - -c + args: + - gzip --stdout /stackable/nifi/flow/flow.json > /stackable/data/database/flow.json.gz + volumeMounts: + - name: nifi-flow + mountPath: /stackable/nifi/flow + - name: database-repository + mountPath: /stackable/data/database + containers: + - name: nifi + ports: + - name: greeting + containerPort: 8090 + protocol: TCP + volumes: + - name: nifi-flow + configMap: +{% if test_scenario['values']['nifi'].startswith("1.") %} + name: nifi1-flow +{% else %} + name: nifi2-flow +{% endif %} + roleGroups: + default: + replicas: 2 +--- +apiVersion: v1 +kind: Service +metadata: + name: nifi-greeting +spec: + selector: + app.kubernetes.io/component: node + app.kubernetes.io/instance: test-nifi + app.kubernetes.io/name: nifi + app.kubernetes.io/role-group: default + ports: + - name: greeting + port: 80 + protocol: TCP + targetPort: greeting +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi1-flow +data: + flow.json: | + { + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [ + { + "identifier": "fd3cf892-0196-1000-0000-000074a555a2", + "instanceIdentifier": "fd3cf892-0196-1000-0000-000074a555a2", + "name": "StackablePrometheusReportingTask", + "type": "org.apache.nifi.reporting.prometheus.PrometheusReportingTask", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-prometheus-nar", + "version": "1.28.1" + }, + "properties": { + "prometheus-reporting-task-metrics-endpoint-port": "8081", + "prometheus-reporting-task-metrics-strategy": "All Components", + "prometheus-reporting-task-instance-id": "${hostname(true)}", + "prometheus-reporting-task-client-auth": "No Authentication", + "prometheus-reporting-task-metrics-send-jvm": "true" + }, + "propertyDescriptors": {}, + "scheduledState": "RUNNING", + "schedulingPeriod": "60 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "componentType": "REPORTING_TASK" + } + ], + "templates": [], + "rootGroup": { + "identifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "instanceIdentifier": "fd37b89e-0196-1000-9750-8b45eabee468", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "b34e789e-1a70-3275-b5c5-f5856b628166", + "instanceIdentifier": "fd432952-0196-1000-0000-000048f10556", + "name": "Greet", + "comments": "", + "position": { + "x": 800.0, + "y": 304.0 + }, + "type": "org.apache.nifi.processors.standard.ReplaceText", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.28.1" + }, + "properties": { + "Regular Expression": "(?s)(^.*$)", + "Replacement Value": "Hello!", + "Evaluation Mode": "Entire text", + "Line-by-Line Evaluation Mode": "All", + "Character Set": "UTF-8", + "Maximum Buffer Size": "1 MB", + "Replacement Strategy": "Always Replace" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 25, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + }, + { + "identifier": "d15c871b-e9c5-38bd-903b-6bf3a9c55765", + "instanceIdentifier": "fd45c24b-0196-1000-0000-00001a549d74", + "name": "HandleHttpResponse", + "comments": "", + "position": { + "x": 800.0, + "y": 752.0 + }, + "type": "org.apache.nifi.processors.standard.HandleHttpResponse", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.28.1" + }, + "properties": { + "HTTP Context Map": "fd3b17c4-0196-1000-0000-000076ed786c", + "HTTP Status Code": "200" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success", + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + }, + { + "identifier": "35d73904-95e1-3dae-81a7-56d0f907ad5c", + "instanceIdentifier": "fd4557bd-0196-1000-ffff-ffffb0a46ce1", + "name": "ShoutProcessor", + "comments": "", + "position": { + "x": 800.0, + "y": 528.0 + }, + "type": "tech.stackable.nifi.processors.sample.ShoutProcessor", + "bundle": { + "group": "tech.stackable.nifi", + "artifact": "nifi-sample-nar", + "version": "1.0.0" + }, + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + }, + { + "identifier": "c5fad835-91fd-31dc-afbf-c37209e03598", + "instanceIdentifier": "fd3a912f-0196-1000-ffff-ffffacaca66e", + "name": "HandleHttpRequest", + "comments": "", + "position": { + "x": 800.0, + "y": 80.0 + }, + "type": "org.apache.nifi.processors.standard.HandleHttpRequest", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.28.1" + }, + "properties": { + "multipart-request-max-size": "1 MB", + "Allow POST": "false", + "Default URL Character Set": "UTF-8", + "Allow DELETE": "false", + "Maximum Threads": "200", + "HTTP Protocols": "HTTP_1_1", + "container-queue-size": "50", + "HTTP Context Map": "fd3b17c4-0196-1000-0000-000076ed786c", + "multipart-read-buffer-size": "512 KB", + "Allow OPTIONS": "false", + "Allowed Paths": "/greeting", + "Allow GET": "true", + "Allow HEAD": "false", + "Listening Port": "8090", + "Client Authentication": "No Authentication", + "Allow PUT": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "3cb76499-9b4a-3199-9697-e05fdb86b38e", + "instanceIdentifier": "fd4532b6-0196-1000-ffff-ffffcb96239f", + "name": "", + "source": { + "id": "c5fad835-91fd-31dc-afbf-c37209e03598", + "type": "PROCESSOR", + "groupId": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "name": "HandleHttpRequest", + "comments": "", + "instanceIdentifier": "fd3a912f-0196-1000-ffff-ffffacaca66e" + }, + "destination": { + "id": "b34e789e-1a70-3275-b5c5-f5856b628166", + "type": "PROCESSOR", + "groupId": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "name": "Greet", + "comments": "", + "instanceIdentifier": "fd432952-0196-1000-0000-000048f10556" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + }, + { + "identifier": "895aecef-a30b-3ad5-993a-6df611e64e0d", + "instanceIdentifier": "fd459382-0196-1000-0000-00000ad9142c", + "name": "", + "source": { + "id": "b34e789e-1a70-3275-b5c5-f5856b628166", + "type": "PROCESSOR", + "groupId": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "name": "Greet", + "comments": "", + "instanceIdentifier": "fd432952-0196-1000-0000-000048f10556" + }, + "destination": { + "id": "35d73904-95e1-3dae-81a7-56d0f907ad5c", + "type": "PROCESSOR", + "groupId": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "name": "ShoutProcessor", + "comments": "", + "instanceIdentifier": "fd4557bd-0196-1000-ffff-ffffb0a46ce1" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + }, + { + "identifier": "47c43ae4-44e5-3e94-9185-bfae5fd6776e", + "instanceIdentifier": "fd45f133-0196-1000-ffff-fffff094098e", + "name": "", + "source": { + "id": "35d73904-95e1-3dae-81a7-56d0f907ad5c", + "type": "PROCESSOR", + "groupId": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "name": "ShoutProcessor", + "comments": "", + "instanceIdentifier": "fd4557bd-0196-1000-ffff-ffffb0a46ce1" + }, + "destination": { + "id": "d15c871b-e9c5-38bd-903b-6bf3a9c55765", + "type": "PROCESSOR", + "groupId": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9", + "name": "HandleHttpResponse", + "comments": "", + "instanceIdentifier": "fd45c24b-0196-1000-0000-00001a549d74" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [ + { + "identifier": "0e58c44e-c294-3b7f-b323-bb94ca5074fb", + "instanceIdentifier": "fd3b17c4-0196-1000-0000-000076ed786c", + "name": "StandardHttpContextMap", + "comments": "", + "type": "org.apache.nifi.http.StandardHttpContextMap", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-http-context-map-nar", + "version": "1.28.1" + }, + "properties": { + "Request Expiration": "1 min", + "Maximum Outstanding Requests": "5000" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [ + { + "type": "org.apache.nifi.http.HttpContextMap", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-services-api-nar", + "version": "1.28.1" + } + } + ], + "scheduledState": "ENABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "b4fe1571-02fd-3351-a2c0-e92f420b3ce9" + } + ], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "componentType": "PROCESS_GROUP" + } + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi2-flow +data: + flow.json: | + { + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "flowAnalysisRules": [], + "rootGroup": { + "identifier": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "instanceIdentifier": "f7cda0ce-0196-1000-30b6-0f5577f07b63", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "238c6f6f-9e6b-3faf-bcdc-7058c9473c76", + "instanceIdentifier": "f80dee83-0196-1000-0000-000051a29aa3", + "name": "ShoutProcessor", + "comments": "", + "position": { + "x": -112.0, + "y": 96.0 + }, + "type": "tech.stackable.nifi.processors.sample.ShoutProcessor", + "bundle": { + "group": "tech.stackable.nifi", + "artifact": "nifi-sample-nar", + "version": "1.0.0" + }, + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + }, + { + "identifier": "1975618d-36bc-32b4-9d14-06a14d4dcba0", + "instanceIdentifier": "f80d8c53-0196-1000-ffff-ffffb40a54f8", + "name": "Greet", + "comments": "", + "position": { + "x": -112.0, + "y": -128.0 + }, + "type": "Greet", + "bundle": { + "group": "org.apache.nifi", + "artifact": "python-extensions", + "version": "1.0.0" + }, + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 25, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "original", + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + }, + { + "identifier": "0b7338da-d770-365b-bff5-bba3f0236229", + "instanceIdentifier": "f80e3b3f-0196-1000-0000-00004590be6a", + "name": "HandleHttpResponse", + "comments": "", + "position": { + "x": -112.0, + "y": 320.0 + }, + "type": "org.apache.nifi.processors.standard.HandleHttpResponse", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "2.2.0" + }, + "properties": { + "HTTP Context Map": "f80ef95f-0196-1000-ffff-ffffabf564cb", + "HTTP Status Code": "200" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success", + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + }, + { + "identifier": "522d46b7-8c43-35b7-ab91-2e5150e44e5b", + "instanceIdentifier": "f809987c-0196-1000-ffff-fffff71b58df", + "name": "HandleHttpRequest", + "comments": "", + "position": { + "x": -112.0, + "y": -352.0 + }, + "type": "org.apache.nifi.processors.standard.HandleHttpRequest", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "2.2.0" + }, + "properties": { + "multipart-request-max-size": "1 MB", + "Allow POST": "false", + "Default URL Character Set": "UTF-8", + "Allow DELETE": "false", + "Request Header Maximum Size": "8 KB", + "Maximum Threads": "200", + "HTTP Protocols": "HTTP_1_1", + "container-queue-size": "50", + "HTTP Context Map": "f80ef95f-0196-1000-ffff-ffffabf564cb", + "multipart-read-buffer-size": "512 KB", + "Allow OPTIONS": "false", + "Allowed Paths": "/greeting", + "Allow GET": "true", + "Allow HEAD": "false", + "Listening Port": "8090", + "Client Authentication": "No Authentication", + "Allow PUT": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "5f7beeba-4cdc-3099-aef6-96ced3560ecc", + "instanceIdentifier": "f80e0681-0196-1000-ffff-ffffc59964fa", + "name": "", + "source": { + "id": "1975618d-36bc-32b4-9d14-06a14d4dcba0", + "type": "PROCESSOR", + "groupId": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "name": "Greet", + "comments": "", + "instanceIdentifier": "f80d8c53-0196-1000-ffff-ffffb40a54f8" + }, + "destination": { + "id": "238c6f6f-9e6b-3faf-bcdc-7058c9473c76", + "type": "PROCESSOR", + "groupId": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "name": "ShoutProcessor", + "comments": "", + "instanceIdentifier": "f80dee83-0196-1000-0000-000051a29aa3" + }, + "labelIndex": 0, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + }, + { + "identifier": "5165254a-e18c-3ccb-b8cb-25c55d7151a7", + "instanceIdentifier": "f80e54a1-0196-1000-ffff-ffffa1c1bed7", + "name": "", + "source": { + "id": "238c6f6f-9e6b-3faf-bcdc-7058c9473c76", + "type": "PROCESSOR", + "groupId": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "name": "ShoutProcessor", + "comments": "", + "instanceIdentifier": "f80dee83-0196-1000-0000-000051a29aa3" + }, + "destination": { + "id": "0b7338da-d770-365b-bff5-bba3f0236229", + "type": "PROCESSOR", + "groupId": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "name": "HandleHttpResponse", + "comments": "", + "instanceIdentifier": "f80e3b3f-0196-1000-0000-00004590be6a" + }, + "labelIndex": 0, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + }, + { + "identifier": "ad7fc77b-6b4b-3750-8e22-fcbb228a6fb4", + "instanceIdentifier": "f80dbacc-0196-1000-0000-0000274b2561", + "name": "", + "source": { + "id": "522d46b7-8c43-35b7-ab91-2e5150e44e5b", + "type": "PROCESSOR", + "groupId": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "name": "HandleHttpRequest", + "comments": "", + "instanceIdentifier": "f809987c-0196-1000-ffff-fffff71b58df" + }, + "destination": { + "id": "1975618d-36bc-32b4-9d14-06a14d4dcba0", + "type": "PROCESSOR", + "groupId": "a0cca604-51c7-32b2-9529-72ec3566e93c", + "name": "Greet", + "comments": "", + "instanceIdentifier": "f80d8c53-0196-1000-ffff-ffffb40a54f8" + }, + "labelIndex": 0, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [ + { + "identifier": "68712a48-5a26-3bee-8bfc-dafafda62660", + "instanceIdentifier": "f80ef95f-0196-1000-ffff-ffffabf564cb", + "name": "StandardHttpContextMap", + "comments": "", + "type": "org.apache.nifi.http.StandardHttpContextMap", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-http-context-map-nar", + "version": "2.2.0" + }, + "properties": { + "Request Expiration": "1 min", + "Maximum Outstanding Requests": "5000" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [ + { + "type": "org.apache.nifi.http.HttpContextMap", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-services-api-nar", + "version": "2.2.0" + } + } + ], + "scheduledState": "ENABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "a0cca604-51c7-32b2-9529-72ec3566e93c" + } + ], + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "scheduledState": "ENABLED", + "executionEngine": "INHERITED", + "maxConcurrentTasks": 1, + "statelessFlowTimeout": "1 min", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "componentType": "PROCESS_GROUP" + } + } diff --git a/tests/templates/kuttl/custom-components-git-sync/40-assert.yaml b/tests/templates/kuttl/custom-components-git-sync/40-assert.yaml new file mode 100644 index 00000000..2c407438 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/40-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-nifi-greeting +status: + succeeded: 1 diff --git a/tests/templates/kuttl/custom-components-git-sync/40-test-nifi-greeting.yaml b/tests/templates/kuttl/custom-components-git-sync/40-test-nifi-greeting.yaml new file mode 100644 index 00000000..300390d1 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/40-test-nifi-greeting.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-nifi-greeting +spec: + template: + spec: + containers: + - name: test + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - /bin/bash + - -c + args: + - test "$(curl nifi-greeting/greeting)" = HELLO! + restartPolicy: OnFailure + terminationGracePeriodSeconds: 1 diff --git a/tests/templates/kuttl/custom-components-git-sync/README.md b/tests/templates/kuttl/custom-components-git-sync/README.md new file mode 100644 index 00000000..67829d52 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/README.md @@ -0,0 +1,36 @@ +# Overview + +This test case configures the NiFi cluster to fetch Python and Java +processors via git-sync. A flow is deployed that include these +processors. The test job triggers the flow and checks the result. + +# Custom processors + +## Python processor "Greet" + +The Greet processor is implemented in Python and answers a request with +the content "Hello!". + +Only NiFi 2 supports Python components. In the flow of NiFi 1, the +Python processor is replaced with the internal ReplaceText processor. +NiFi 1 ignores the Python configurations. + +## Java processor "Shout" + +The Shout processor is implemented in Java and transforms the received +content to uppercase. + +The processor is compiled with the dependencies for NiFi 2, but also +works in NiFi 1 if compiled with JDK 11. + +# Flow + +![NiFi canvas](./canvas.png) + +1. The HandleHttpRequest processor listens for HTTP requests to + `GET /greeting`. The request is forwarded to the Greet processor. +2. The Greet processor writes the content "Hello!" to the flow file that + is then forwarded to the Shout processor. +3. The Shout processor transforms the content to uppercase. The flow + file is forwarded to the HandleHttpResponse processor. +4. The HandleHttpResponse answers the HTTP request with this content. diff --git a/tests/templates/kuttl/custom-components-git-sync/canvas.png b/tests/templates/kuttl/custom-components-git-sync/canvas.png new file mode 100644 index 00000000..e6872596 Binary files /dev/null and b/tests/templates/kuttl/custom-components-git-sync/canvas.png differ diff --git a/tests/templates/kuttl/custom-components-git-sync/java-processors/nifi-sample-nar-1.0.0.nar b/tests/templates/kuttl/custom-components-git-sync/java-processors/nifi-sample-nar-1.0.0.nar new file mode 100644 index 00000000..bd203bf8 Binary files /dev/null and b/tests/templates/kuttl/custom-components-git-sync/java-processors/nifi-sample-nar-1.0.0.nar differ diff --git a/tests/templates/kuttl/custom-components-git-sync/java-processors/sample-processor.tar.gz b/tests/templates/kuttl/custom-components-git-sync/java-processors/sample-processor.tar.gz new file mode 100644 index 00000000..f599ac0e Binary files /dev/null and b/tests/templates/kuttl/custom-components-git-sync/java-processors/sample-processor.tar.gz differ diff --git a/tests/templates/kuttl/custom-components-git-sync/python-processors/greet_processor.py b/tests/templates/kuttl/custom-components-git-sync/python-processors/greet_processor.py new file mode 100644 index 00000000..e646e659 --- /dev/null +++ b/tests/templates/kuttl/custom-components-git-sync/python-processors/greet_processor.py @@ -0,0 +1,16 @@ +from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult + + +class Greet(FlowFileTransform): + class Java: + implements = ["org.apache.nifi.python.processor.FlowFileTransform"] + + class ProcessorDetails: + version = "1.0.0" + description = "A Python processor that greets politely." + + def __init__(self, **kwargs): + pass + + def transform(self, context, flowfile): + return FlowFileTransformResult(relationship="success", contents="Hello!") diff --git a/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 b/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 index c3926671..a04ef2a4 100644 --- a/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 @@ -96,6 +96,10 @@ spec: - authenticationClass: simple-nifi-users sensitiveProperties: keySecret: nifi-sensitive-property-key + customComponentsGitSync: + - repo: https://github.com/stackabletech/nifi-operator + branch: ef61c87311ad2f57484c33245c9ed50908a1c785 + gitFolder: tests/templates/kuttl/custom-processors-git-sync/processors-0 vectorAggregatorConfigMapName: nifi-vector-aggregator-discovery zookeeperConfigMapName: test-nifi-znode nodes: @@ -116,6 +120,14 @@ spec: loggers: ROOT: level: INFO + git-sync: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO vector: console: level: INFO diff --git a/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 index 3a16c399..1c4d5f31 100644 --- a/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 @@ -30,6 +30,18 @@ customConfig: condition: >- .pod == "test-nifi-node-automatic-log-config-0" && .container == "nifi" + filteredAutomaticLogConfigNodeGitSync: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-nifi-node-automatic-log-config-0" && + .container == "git-sync-0" + filteredAutomaticLogConfigNodeGitSyncInit: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-nifi-node-automatic-log-config-0" && + .container == "git-sync-0-init" filteredAutomaticLogConfigNodeVector: type: filter inputs: [validEvents] diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index d11c712d..196553b7 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -147,6 +147,11 @@ tests: - iceberg-use-kerberos - kerberos-realm - openshift + - name: custom-components-git-sync + dimensions: + - nifi + - zookeeper-latest + - openshift suites: - name: nightly patch: