BIGTOP-4131: Add Tez component on Bigtop-3.3.0 stack

bigtop-manager-agent/src/main/resources/bin/start.sh

@@ -23,12 +23,41 @@ BIGTOP_MANAGER_HOME=${BIGTOP_MANAGER_HOME:-$(cd $BIN_DIR/..; pwd)}
 
 source "${BIGTOP_MANAGER_HOME}/bin/env.sh"
 
+usage() {
+    echo "usage: $PROG [--debug]"
+    echo "       --debug        - enable debug mode"
+    echo "       -h, --help"
+    exit 1
+}
+
+DEBUG="false"
+DOCKER="false"
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+    --debug)
+        echo "enable debug mode."
+        DEBUG="true"
+        shift;;
+    -h|--help)
+        usage
+        shift;;
+    *)
+        echo "Unknown argument: '$1'" 1>&2
+        usage;;
+    esac
+done
+
 JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
 
 if [[ "$DOCKER" == "true" ]]; then
   JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
 fi
 
+if [[ "$DEBUG" == "true" ]]; then
+  JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006"
+fi
+
 cd $BIGTOP_MANAGER_HOME
 
 $JAVA_HOME/bin/java $JAVA_OPTS \

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java

@@ -42,14 +42,21 @@ private Constants() {
      */
     public static final String ALL_HOST_KEY = "all";
 
+    /**
+     * permission 644
+     */
+    public static final String PERMISSION_644 = "rw-r--r--";
+
     /**
      * permission 755
      */
     public static final String PERMISSION_755 = "rwxr-xr-x";
+
     /**
-     * permission 644
+     * permission 775
      */
-    public static final String PERMISSION_644 = "rw-r--r--";
+    public static final String PERMISSION_775 = "rwxrwxr-x";
+
     /**
      * permission 777
      */

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java

@@ -28,13 +28,15 @@
 import org.apache.bigtop.manager.dao.repository.HostComponentRepository;
 import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage;
 import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage;
+import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage;
 import org.apache.bigtop.manager.server.command.stage.ComponentInstallStage;
 import org.apache.bigtop.manager.server.command.stage.ComponentStartStage;
 import org.apache.bigtop.manager.server.command.stage.ComponentStopStage;
 import org.apache.bigtop.manager.server.command.stage.StageContext;
 import org.apache.bigtop.manager.server.exception.ServerException;
 import org.apache.bigtop.manager.server.holder.SpringContextHolder;
 import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
+import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO;
 import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
 import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO;
 import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper;
@@ -178,6 +180,23 @@ protected void createInstallStages() {
         }
     }
 
+    protected void createConfigureStages() {
+        for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) {
+            if (serviceCommand.getInstalled()) {
+                continue;
+            }
+
+            for (ComponentHostDTO componentHost : serviceCommand.getComponentHosts()) {
+                String serviceName = serviceCommand.getServiceName();
+                String componentName = componentHost.getComponentName();
+                List<String> hostnames = componentHost.getHostnames();
+
+                StageContext stageContext = createStageContext(serviceName, componentName, hostnames);
+                stages.add(new ComponentConfigureStage(stageContext));
+            }
+        }
+    }
+
     protected void createStartStages() {
         List<String> todoList = getTodoListForCommand(Command.START);
 

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java

@@ -26,7 +26,15 @@ public ServiceConfigureJob(JobContext jobContext) {
 
     @Override
     protected void createStages() {
+        // Update cache files
         super.createCacheStage();
+
+        // Configure services
+        super.createConfigureStages();
+
+        // Restart services
+        super.createStopStages();
+        super.createStartStages();
     }
 
     @Override

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java

@@ -67,6 +67,9 @@ protected void createStages() {
         // Update cache files after installed
         super.createCacheStage();
 
+        // Configure services
+        super.createConfigureStages();
+
         // Start all master components
         super.createStartStages();
 

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.command.stage;
+
+import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask;
+import org.apache.bigtop.manager.server.command.task.Task;
+
+public class ComponentConfigureStage extends AbstractComponentStage {
+
+    public ComponentConfigureStage(StageContext stageContext) {
+        super(stageContext);
+    }
+
+    @Override
+    protected Task createTask(String hostname) {
+        return new ComponentConfigureTask(createTaskContext(hostname));
+    }
+
+    @Override
+    public String getName() {
+        return "Configure " + stageContext.getComponentDTO().getDisplayName();
+    }
+}

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.command.task;
+
+import org.apache.bigtop.manager.common.enums.Command;
+
+public class ComponentConfigureTask extends AbstractComponentTask {
+
+    public ComponentConfigureTask(TaskContext taskContext) {
+        super(taskContext);
+    }
+
+    @Override
+    protected Command getCommand() {
+        return Command.CONFIGURE;
+    }
+
+    @Override
+    public String getName() {
+        return "Configure " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname();
+    }
+}

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java

@@ -33,9 +33,6 @@
 @XmlAccessorType(XmlAccessType.FIELD)
 public class ConfigurationXml {
 
-    @XmlElement(name = "schema-version")
-    private String schemaVersion;
-
     @XmlElement(name = "property")
     private List<PropertyModel> propertyModels;
 }

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java

@@ -24,16 +24,12 @@
 
 import jakarta.xml.bind.annotation.XmlAccessType;
 import jakarta.xml.bind.annotation.XmlAccessorType;
-import jakarta.xml.bind.annotation.XmlElement;
 import jakarta.xml.bind.annotation.XmlRootElement;
 
 @Data
 @XmlRootElement(name = "metainfo")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class ServiceMetainfoXml {
 
-    @XmlElement(name = "schema-version")
-    private String schemaVersion;
-
     private ServiceModel service;
 }

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java

@@ -32,9 +32,6 @@
 @XmlAccessorType(XmlAccessType.FIELD)
 public class StackMetainfoXml {
 
-    @XmlElement(name = "schema-version")
-    private String schemaVersion;
-
     @XmlElement(name = "stack")
     private StackModel stack;
 }

bigtop-manager-server/src/main/resources/bin/start.sh

@@ -58,7 +58,6 @@ if [[ "$DEBUG" == "true" ]]; then
   JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005"
 fi
 
-
 cd $BIGTOP_MANAGER_HOME
 
 $JAVA_HOME/bin/java $JAVA_OPTS \

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml

@@ -19,7 +19,6 @@
 -->
 
 <metainfo>
-    <schema-version>2.0</schema-version>
     <stack>
         <stack-name>bigtop</stack-name>
         <stack-version>3.3.0</stack-version>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml

@@ -86,33 +86,22 @@
 
 USER="$(whoami)"
 export JAVA_HOME=${java_home!}
-
 export HADOOP_HOME=${hadoop_home!}
-
 export HADOOP_COMMON_HOME=${hadoop_home!}
-
 export HADOOP_HDFS_HOME=${hadoop_hdfs_home!}
-
 export HADOOP_CONF_DIR=${hadoop_conf_dir!}
-
 export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max}
-
 export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
 
-export HADOOP_JAAS_DEBUG=true
-
 # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"
 
 <#noparse>
 export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
 </#noparse>
 
 export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER
-
 export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER
-
 export HADOOP_ROOT_LOGGER=${hadoop_root_logger}
-
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
 
 ]]>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml

@@ -19,7 +19,6 @@
 -->
 
 <metainfo>
-    <schema-version>2.0</schema-version>
     <service>
         <name>hdfs</name>
         <display-name>HDFS</display-name>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml

@@ -19,7 +19,6 @@
 -->
 
 <metainfo>
-    <schema-version>2.0</schema-version>
     <service>
         <name>kafka</name>
         <display-name>Kafka</display-name>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json

@@ -7,8 +7,5 @@
   ],
   "KAFKA_BROKER-RESTART": [
     "ZOOKEEPER_SERVER-RESTART"
-  ],
-  "ZOOKEEPER_SERVER-STOP": [
-    "KAFKA_BROKER-STOP"
   ]
 }

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<configuration>
+    <!-- tez-env.sh -->
+    <property>
+        <name>content</name>
+        <display-name>tez-env template</display-name>
+        <description>This is the freemarker template for tez-env.sh file</description>
+        <value><![CDATA[
+# Tez specific configuration
+export TEZ_CONF_DIR=${tez_conf_dir!}
+
+# Set HADOOP_HOME to point to a specific hadoop install directory
+export HADOOP_HOME=${hadoop_home!}
+
+# The java implementation to use.
+export JAVA_HOME=${java_home!}
+]]>
+        </value>
+        <attrs>
+            <type>longtext</type>
+        </attrs>
+    </property>
+    <property>
+        <name>enable_heap_dump</name>
+        <value>false</value>
+        <description>Enable or disable taking Heap Dump. (true/false)</description>
+        <display-name>Enable heap dump</display-name>
+    </property>
+    <property>
+        <name>heap_dump_location</name>
+        <value>/tmp</value>
+        <description>Location for heap dump file</description>
+        <display-name>Heap dump location</display-name>
+    </property>
+</configuration>