From 4045f1fdbeb5cd81b90308a5ef63e1b58167b958 Mon Sep 17 00:00:00 2001 From: Julien Guitton Date: Thu, 11 Jan 2024 12:58:18 +0100 Subject: [PATCH 1/3] Update bootstrap.conf template --- .../templates/config/bootstrap_properties.go | 56 ++++++++++++++++--- site/docs/4_compatibility_versions.md | 6 +- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/pkg/resources/templates/config/bootstrap_properties.go b/pkg/resources/templates/config/bootstrap_properties.go index 1717cf94f0..7027ebc4af 100644 --- a/pkg/resources/templates/config/bootstrap_properties.go +++ b/pkg/resources/templates/config/bootstrap_properties.go @@ -23,6 +23,9 @@ java=java # Username to use when running NiFi. This value will be ignored on Windows. run.as= +# Preserve shell environment while runnning as "run.as" user +preserve.environment=false + # Configure where NiFi's lib and conf directories live lib.dir=./lib conf.dir=./conf @@ -40,25 +43,57 @@ java.arg.3=-Xmx{{.JvmMemory}} # Enable Remote Debugging #java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 -java.arg.4=-Djava.net.preferIPv4Stack=true - # allowRestrictedHeaders is required for Cluster/Node communications to work properly java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - #Set headless mode by default java.arg.14=-Djava.awt.headless=true -# Master key in hexadecimal format for encrypted sensitive configuration values +# Root key in hexadecimal format for encrypted sensitive configuration values nifi.bootstrap.sensitive.key= +# Sensitive Property Provider configuration + +# HashiCorp Vault Sensitive Property Providers +#nifi.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf + +# AWS Sensitive Property Providers +#nifi.bootstrap.protection.aws.conf=./conf/bootstrap-aws.conf + +# Azure Key Vault Sensitive Property Providers +#nifi.bootstrap.protection.azure.keyvault.conf=./conf/bootstrap-azure.conf + +# GCP KMS Sensitive Property Providers +#nifi.bootstrap.protection.gcp.kms.conf=./conf/bootstrap-gcp.conf + # Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs java.arg.15=-Djava.security.egd=file:/dev/urandom +# Requires JAAS to use only the provided JAAS configuration to authenticate a Subject, without using any "fallback" methods (such as prompting for username/password) +# Please see https://docs.oracle.com/en/java/javase/17/security/single-sign-using-kerberos-java1.html, section "EXCEPTIONS TO THE MODEL" +java.arg.16=-Djavax.security.auth.useSubjectCredsOnly=true + +# Zookeeper 3.5 now includes an Admin Server that starts on port 8080, since NiFi is already using that port disable by default. +# Please see https://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_adminserver_config for configuration options. +java.arg.17=-Dzookeeper.admin.enableServer=false + +# The following options configure a Java Agent to handle native library loading. +# It is needed when a custom jar (eg. JDBC driver) has been configured on a component in the flow and this custom jar depends on a native library +# and tries to load it by its absolute path (java.lang.System.load(String filename) method call). +# Use this Java Agent only if you get "Native Library ... already loaded in another classloader" errors otherwise! +#java.arg.18=-javaagent:./lib/aspectj/aspectjweaver-1.9.20.1.jar +#java.arg.19=-Daj.weaving.loadersToSkip=sun.misc.Launcher$AppClassLoader,jdk.internal.loader.ClassLoaders$AppClassLoader,org.eclipse.jetty.webapp.WebAppClassLoader,\ +# org.apache.jasper.servlet.JasperLoader,org.jvnet.hk2.internal.DelegatingClassLoader,org.apache.nifi.nar.NarClassLoader +# End of Java Agent config for native library loading. + +# The following entry is needed in Java 21 because some libraries invoke +# reflective calls that Java no longer considers allowed by default. +# https://docs.oracle.com/en/java/javase/16/migrate/migrating-jdk-8-later-jdk-releases.html#GUID-7BB28E4D-99B3-4078-BDC4-FC24180CE82B +# This may need to be modified if additional reflective access is needed by certain libraries +# This is only known to be needed for the Hive3 processors as of now. +java.arg.20=--add-opens=java.base/java.net=ALL-UNNAMED + ### # Notification Services for notifying interested parties when NiFi is stopped, started, dies ### @@ -77,4 +112,11 @@ notification.max.attempts=5 # Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? #nifi.dead.notification.services=email-notification + +# The first curator connection issue is logged as ERROR, for example when NiFi cannot connect to one of the Zookeeper nodes. +# Additional connection issues are logged as DEBUG until the connection is restored. +java.arg.curator.supress.excessive.logs=-Dcurator-log-only-first-connection-issue-as-error-level=true + +# Port used to listen for communications from NiFi. If this property is missing, empty, or 0, a random ephemeral port is used. +nifi.bootstrap.listen.port=0 ` diff --git a/site/docs/4_compatibility_versions.md b/site/docs/4_compatibility_versions.md index d8d0cdde37..5a3f0421f5 100644 --- a/site/docs/4_compatibility_versions.md +++ b/site/docs/4_compatibility_versions.md @@ -13,9 +13,9 @@ Nifikop supports the following NiFi cluster features: | NiFi Version | Cluster deployment | Standalone deployment | Cluster nodes configuration | Cluster rolling upgrade | Cluster scaling | Cluster auto-scaling | Prometheus Reporting | |--------------|--------------------|-----------------------|-----------------------------|-------------------------|-----------------|----------------------|----------------------| -| NiFi 1.16 | Yes | No | Yes | Yes | Yes | Yes | Yes | -| NiFi 1.17 | Yes | No | Yes | Yes | Yes | Yes | Yes | -| NiFi 1.18 | Yes | No | Yes | Yes | Yes | Yes | Yes | +| NiFi 1.16 | No | No | No | Yes | Yes | Yes | Yes | +| NiFi 1.17 | No | No | No | Yes | Yes | Yes | Yes | +| NiFi 1.18 | No | No | No | Yes | Yes | Yes | Yes | | NiFi 1.19 | Yes | No | Yes | Yes | Yes | Yes | Yes | | NiFi 1.20 | Yes | No | Yes | Yes | Yes | Yes | Yes | | NiFi 1.21 | Yes | No | Yes | Yes | Yes | Yes | Yes | From 49ebf72da6acf1148a2249f0deda1a6ad620de99 Mon Sep 17 00:00:00 2001 From: Julien Guitton Date: Thu, 11 Jan 2024 13:04:23 +0100 Subject: [PATCH 2/3] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8838e510b..650faa208a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,14 @@ - [PR #349](https://github.com/konpyutaika/nifikop/pull/349) - **[Operator/NifiRegistryClient]** Set FlowRegistry type in RegistryClient creation. - [PR #350](https://github.com/konpyutaika/nifikop/pull/350) - **[Operator]** Remove optimistic lock on `Patch`. - [PR #352](https://github.com/konpyutaika/nifikop/pull/352) - **[Operator]** Changed default LogLevel of NiFi from `DEBUG` to `INFO`. +- [PR #355](https://github.com/konpyutaika/nifikop/pull/355) - **[Operator/NifiCluster]** Updated `bootstrap.conf` template for 2.0.0-M1. ### Fixed Bugs ### Deprecated +- [PR #355](https://github.com/konpyutaika/nifikop/pull/355) - **[Operator/NifiCluster]** Removed ability to deploy NiFi cluster in version `<=1.18`. + ### Removed ## v1.6.0 From 2a68789390f4d03c41d095cad65528f1f016988f Mon Sep 17 00:00:00 2001 From: Julien Guitton Date: Thu, 11 Jan 2024 16:15:07 +0100 Subject: [PATCH 3/3] Make bootstrap.conf template optional --- CHANGELOG.md | 1 + pkg/resources/nifi/secretconfig.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fae665340d..b75a90f618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - [PR #352](https://github.com/konpyutaika/nifikop/pull/352) - **[Operator]** Changed default LogLevel of NiFi from `DEBUG` to `INFO`. - [PR #354](https://github.com/konpyutaika/nifikop/pull/354) - **[Operator/NifiCluster]** Updated `login_identity_providers.xml` template for 2.0.0-M1. - [PR #355](https://github.com/konpyutaika/nifikop/pull/355) - **[Operator/NifiCluster]** Updated `bootstrap.conf` template for 2.0.0-M1. +- [PR #355](https://github.com/konpyutaika/nifikop/pull/355) - **[Operator/NifiCluster]** Updated `bootstrap.conf` template logic to only use it if needed. ### Fixed Bugs diff --git a/pkg/resources/nifi/secretconfig.go b/pkg/resources/nifi/secretconfig.go index 53fcb4fdc0..12033df2db 100644 --- a/pkg/resources/nifi/secretconfig.go +++ b/pkg/resources/nifi/secretconfig.go @@ -44,7 +44,6 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas "state-management.xml": []byte(r.getStateManagementConfigString(nodeConfig, id, log)), "login-identity-providers.xml": []byte(r.getLoginIdentityProvidersConfigString(nodeConfig, id, log)), "logback.xml": []byte(r.getLogbackConfigString(nodeConfig, id, log)), - "bootstrap.conf": []byte(r.generateBootstrapPropertiesNodeConfig(id, nodeConfig, log)), "bootstrap-notification-services.xml": []byte(r.getBootstrapNotificationServicesConfigString(nodeConfig, id, log)), }, } @@ -52,6 +51,10 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas if configcommon.UseSSL(r.NifiCluster) { secret.Data["authorizers.xml"] = []byte(r.getAuthorizersConfigString(nodeConfig, id, log)) } + if bootstrapPropertiesNodeConfig := r.generateBootstrapPropertiesNodeConfig(id, nodeConfig, log); bootstrapPropertiesNodeConfig != nil { + secret.Data["bootstrap.conf"] = []byte(*bootstrapPropertiesNodeConfig) + } + return secret } @@ -527,7 +530,7 @@ func (r *Reconciler) getAuthorizersConfigString(nConfig *v1.NodeConfig, id int32 // Bootstrap properties configuration // ///////////////////////////////////////// -func (r Reconciler) generateBootstrapPropertiesNodeConfig(id int32, nodeConfig *v1.NodeConfig, log zap.Logger) string { +func (r Reconciler) generateBootstrapPropertiesNodeConfig(id int32, nodeConfig *v1.NodeConfig, log zap.Logger) *string { var readOnlyClusterConfig map[string]string if &r.NifiCluster.Spec.ReadOnlyConfig != (&v1.ReadOnlyConfig{}) && &r.NifiCluster.Spec.ReadOnlyConfig.BootstrapProperties != (&v1.BootstrapProperties{}) { @@ -568,6 +571,10 @@ func (r Reconciler) generateBootstrapPropertiesNodeConfig(id int32, nodeConfig * zap.Error(err)) } + if len(completeConfigMap) == 0 { + return nil + } + if err := mergo.Merge(&completeConfigMap, util.ParsePropertiesFormat(r.getBootstrapPropertiesConfigString(nodeConfig, id, log))); err != nil { log.Error("error occurred during merging operator generated configs", zap.String("clusterName", r.NifiCluster.Name), @@ -584,7 +591,8 @@ func (r Reconciler) generateBootstrapPropertiesNodeConfig(id int32, nodeConfig * // We need to sort the config every time to avoid diffs occurred because of ranging through map sort.Strings(completeConfig) - return strings.Join(completeConfig, "\n") + output := strings.Join(completeConfig, "\n") + return &output } func (r *Reconciler) getBootstrapPropertiesConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string {