BIGTOP-4124: Add YARN component on Bigtop-3.3.0 stack (#16)

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml

@@ -167,11 +167,11 @@
     </property>
     <property>
         <name>fs.azure.user.agent.prefix</name>
-        <value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/{{version}}</value>
+        <value/>
     </property>
     <property>
         <name>fs.gs.application.name.suffix</name>
-        <value>(GPN:Hortonworks; version 1.0) HDP/{{version}}</value>
+        <value/>
     </property>
     <property>
         <name>fs.gs.path.encoding</name>
@@ -184,6 +184,6 @@
     </property>
     <property>
         <name>fs.s3a.user.agent.prefix</name>
-        <value>User-Agent: APN/1.0 Hortonworks/1.0 HDP/{{version}}</value>
+        <value/>
     </property>
 </configuration>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml

@@ -24,7 +24,7 @@
         <name>hdfs</name>
         <display-name>HDFS</display-name>
         <desc>Apache Hadoop Distributed File System.</desc>
-        <version>3.3.5</version>
+        <version>3.3.6-1</version>
         <user>hdfs</user>
         <group>hdfs</group>
 

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml

@@ -28,7 +28,7 @@
             high-performance data pipelines, streaming analytics, data integration, and mission-critical
             applications.
         </desc>
-        <version>2.8.2</version>
+        <version>2.8.2-1</version>
         <user>kafka</user>
         <group>kafka</group>
 

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<configuration>
+    <property>
+        <name>content</name>
+        <display-name>hdfs slaves template</display-name>
+        <description>This is the freemarker template for hdfs file</description>
+        <value><![CDATA[
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<#if exclude_hosts?? >
+<#list exclude_hosts as host>
+${host}
+</#list>
+</#if>
+
+]]>
+        </value>
+    </property>
+</configuration>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-env.xml

@@ -0,0 +1,215 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+<configuration>
+    <property>
+        <name>yarn_log_dir_prefix</name>
+        <value>/var/log/hadoop-yarn</value>
+        <display-name>YARN Log Dir Prefix</display-name>
+        <description>YARN Log Dir Prefix</description>
+    </property>
+    <property>
+        <name>yarn_pid_dir_prefix</name>
+        <value>/var/run/hadoop-yarn</value>
+        <display-name>YARN PID Dir Prefix</display-name>
+        <description>YARN PID Dir Prefix</description>
+    </property>
+    <property>
+        <name>yarn_heapsize</name>
+        <value>1024</value>
+        <display-name>YARN Java heap size</display-name>
+        <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
+    </property>
+    <property>
+        <name>resourcemanager_heapsize</name>
+        <value>1024</value>
+        <display-name>ResourceManager Java heap size</display-name>
+        <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
+    </property>
+    <property>
+        <name>nodemanager_heapsize</name>
+        <value>1024</value>
+        <display-name>NodeManager Java heap size</display-name>
+        <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
+    </property>
+    <property>
+        <name>min_user_id</name>
+        <value>1000</value>
+        <display-name>Minimum user ID for submitting job</display-name>
+        <description>Set to 0 to disallow root from submitting jobs. Set to 1000 to disallow all superusers from submitting jobs</description>
+    </property>
+    <property>
+        <name>is_supported_yarn_ranger</name>
+        <value>false</value>
+        <description>Set to false by default,  needs to be set to true in stacks that use Ranger Yarn Plugin</description>
+    </property>
+    <property>
+        <name>yarn_user_nofile_limit</name>
+        <value>32768</value>
+        <description>Max open files limit setting for YARN user.</description>
+    </property>
+    <property>
+        <name>yarn_user_nproc_limit</name>
+        <value>65536</value>
+        <description>Max number of processes limit setting for YARN user.</description>
+    </property>
+    <property>
+        <name>hadoop_java_io_tmpdir</name>
+        <value>/tmp/hadoop-yarn</value>
+        <description>Max number of processes limit setting for YARN user.</description>
+    </property>
+    <!-- yarn-env.sh -->
+    <property>
+        <name>content</name>
+        <display-name>yarn-env template</display-name>
+        <description>This is the freemarker template for yarn-env.sh file</description>
+        <value><![CDATA[
+export HADOOP_HOME=${hadoop_home}
+export HADOOP_YARN_HOME=${hadoop_yarn_home}
+export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
+USER="$(whoami)"
+export HADOOP_LOG_DIR=${yarn_log_dir_prefix}/$USER
+export HADOOP_PID_DIR=${yarn_pid_dir_prefix}/$USER
+export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
+export JAVA_HOME=${java_home}
+export JAVA_LIBRARY_PATH="<#noparse>${JAVA_LIBRARY_PATH}</#noparse>:${hadoop_java_io_tmpdir}"
+<#noparse>
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+
+# resolve links - $0 may be a softlink
+export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+#echo "run java in $JAVA_HOME"
+JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+echo "Error: JAVA_HOME is not set."
+exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+</#noparse>
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE=${yarn_heapsize}
+
+<#noparse>
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+</#noparse>
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_RESOURCEMANAGER_HEAPSIZE=${resourcemanager_heapsize}
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_NODEMANAGER_HEAPSIZE=${nodemanager_heapsize}
+
+# Specify the max Heapsize for the HistoryManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1024.
+# This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+# and/or YARN_HISTORYSERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_HISTORYSERVER_HEAPSIZE=${apptimelineserver_heapsize}
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+<#noparse>
+# default log directory and file
+if [ "$HADOOP_LOG_DIR" = "" ]; then
+HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$HADOOP_LOGFILE" = "" ]; then
+HADOOP_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT"
+export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+</#noparse>
+HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir=${hadoop_java_io_tmpdir}"
+HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
+]]>
+        </value>
+    </property>
+    <property>
+        <name>service_check.queue.name</name>
+        <value>default</value>
+        <description>
+            The queue that used by service check.
+        </description>
+    </property>
+</configuration>