diff --git a/bigtop-manager-agent/src/main/resources/bin/start.sh b/bigtop-manager-agent/src/main/resources/bin/start.sh index 1a6f9cc6..638b5b13 100755 --- a/bigtop-manager-agent/src/main/resources/bin/start.sh +++ b/bigtop-manager-agent/src/main/resources/bin/start.sh @@ -23,12 +23,41 @@ BIGTOP_MANAGER_HOME=${BIGTOP_MANAGER_HOME:-$(cd $BIN_DIR/..; pwd)} source "${BIGTOP_MANAGER_HOME}/bin/env.sh" +usage() { + echo "usage: $PROG [--debug]" + echo " --debug - enable debug mode" + echo " -h, --help" + exit 1 +} + +DEBUG="false" +DOCKER="false" + +while [ $# -gt 0 ]; do + case "$1" in + --debug) + echo "enable debug mode." + DEBUG="true" + shift;; + -h|--help) + usage + shift;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done + JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" fi +if [[ "$DEBUG" == "true" ]]; then + JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006" +fi + cd $BIGTOP_MANAGER_HOME $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java index 66c12b96..14fecb1b 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java @@ -42,14 +42,21 @@ private Constants() { */ public static final String ALL_HOST_KEY = "all"; + /** + * permission 644 + */ + public static final String PERMISSION_644 = "rw-r--r--"; + /** * permission 755 */ public static final String PERMISSION_755 = "rwxr-xr-x"; + /** - * permission 644 + * permission 775 */ - public static final String PERMISSION_644 = "rw-r--r--"; + public static final String PERMISSION_775 = "rwxrwxr-x"; + /** * permission 777 */ diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index 1315077e..4a51ba49 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -28,6 +28,7 @@ import org.apache.bigtop.manager.dao.repository.HostComponentRepository; import org.apache.bigtop.manager.server.command.stage.CacheFileUpdateStage; import org.apache.bigtop.manager.server.command.stage.ComponentCheckStage; +import org.apache.bigtop.manager.server.command.stage.ComponentConfigureStage; import org.apache.bigtop.manager.server.command.stage.ComponentInstallStage; import org.apache.bigtop.manager.server.command.stage.ComponentStartStage; import org.apache.bigtop.manager.server.command.stage.ComponentStopStage; @@ -35,6 +36,7 @@ import org.apache.bigtop.manager.server.exception.ServerException; import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.ComponentDTO; +import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper; @@ -178,6 +180,23 @@ protected void createInstallStages() { } } + protected void createConfigureStages() { + for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { + if (serviceCommand.getInstalled()) { + continue; + } + + for (ComponentHostDTO componentHost : serviceCommand.getComponentHosts()) { + String serviceName = serviceCommand.getServiceName(); + String componentName = componentHost.getComponentName(); + List hostnames = componentHost.getHostnames(); + + StageContext stageContext = createStageContext(serviceName, componentName, hostnames); + stages.add(new ComponentConfigureStage(stageContext)); + } + } + } + protected void createStartStages() { List todoList = getTodoListForCommand(Command.START); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java index 9dad58ac..6f60fc6f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceConfigureJob.java @@ -26,7 +26,15 @@ public ServiceConfigureJob(JobContext jobContext) { @Override protected void createStages() { + // Update cache files super.createCacheStage(); + + // Configure services + super.createConfigureStages(); + + // Restart services + super.createStopStages(); + super.createStartStages(); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java index d97b2094..dd5c727a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceInstallJob.java @@ -67,6 +67,9 @@ protected void createStages() { // Update cache files after installed super.createCacheStage(); + // Configure services + super.createConfigureStages(); + // Start all master components super.createStartStages(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java new file mode 100644 index 00000000..08eea2ad --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.stage; + +import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask; +import org.apache.bigtop.manager.server.command.task.Task; + +public class ComponentConfigureStage extends AbstractComponentStage { + + public ComponentConfigureStage(StageContext stageContext) { + super(stageContext); + } + + @Override + protected Task createTask(String hostname) { + return new ComponentConfigureTask(createTaskContext(hostname)); + } + + @Override + public String getName() { + return "Configure " + stageContext.getComponentDTO().getDisplayName(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java new file mode 100644 index 00000000..553d35ab --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentConfigureTask.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.command.task; + +import org.apache.bigtop.manager.common.enums.Command; + +public class ComponentConfigureTask extends AbstractComponentTask { + + public ComponentConfigureTask(TaskContext taskContext) { + super(taskContext); + } + + @Override + protected Command getCommand() { + return Command.CONFIGURE; + } + + @Override + public String getName() { + return "Configure " + taskContext.getComponentDisplayName() + " on " + taskContext.getHostname(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java index 07f6a106..7981699d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ConfigurationXml.java @@ -33,9 +33,6 @@ @XmlAccessorType(XmlAccessType.FIELD) public class ConfigurationXml { - @XmlElement(name = "schema-version") - private String schemaVersion; - @XmlElement(name = "property") private List propertyModels; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java index cd8fb560..b8c352e5 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/ServiceMetainfoXml.java @@ -24,7 +24,6 @@ import jakarta.xml.bind.annotation.XmlAccessType; import jakarta.xml.bind.annotation.XmlAccessorType; -import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlRootElement; @Data @@ -32,8 +31,5 @@ @XmlAccessorType(XmlAccessType.FIELD) public class ServiceMetainfoXml { - @XmlElement(name = "schema-version") - private String schemaVersion; - private ServiceModel service; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java index 341589b7..65ea7009 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/xml/StackMetainfoXml.java @@ -32,9 +32,6 @@ @XmlAccessorType(XmlAccessType.FIELD) public class StackMetainfoXml { - @XmlElement(name = "schema-version") - private String schemaVersion; - @XmlElement(name = "stack") private StackModel stack; } diff --git a/bigtop-manager-server/src/main/resources/bin/start.sh b/bigtop-manager-server/src/main/resources/bin/start.sh index 855eea8f..7769d50b 100755 --- a/bigtop-manager-server/src/main/resources/bin/start.sh +++ b/bigtop-manager-server/src/main/resources/bin/start.sh @@ -58,7 +58,6 @@ if [[ "$DEBUG" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" fi - cd $BIGTOP_MANAGER_HOME $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml index 78099df7..ea143593 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 bigtop 3.3.0 diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml index b0852040..01ec6084 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml @@ -86,21 +86,13 @@ USER="$(whoami)" export JAVA_HOME=${java_home!} - export HADOOP_HOME=${hadoop_home!} - export HADOOP_COMMON_HOME=${hadoop_home!} - export HADOOP_HDFS_HOME=${hadoop_hdfs_home!} - export HADOOP_CONF_DIR=${hadoop_conf_dir!} - export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max} - export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min} -export HADOOP_JAAS_DEBUG=true - # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug" <#noparse> @@ -108,11 +100,8 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER - export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER - export HADOOP_ROOT_LOGGER=${hadoop_root_logger} - export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir} ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml index c2fc5188..26f77bdb 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 hdfs HDFS diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml index 5e379f3b..844ba3ee 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 kafka Kafka diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json index 78adc63d..09585eb1 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/order.json @@ -7,8 +7,5 @@ ], "KAFKA_BROKER-RESTART": [ "ZOOKEEPER_SERVER-RESTART" - ], - "ZOOKEEPER_SERVER-STOP": [ - "KAFKA_BROKER-STOP" ] } \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml new file mode 100644 index 00000000..0f9c9ffa --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-env.xml @@ -0,0 +1,54 @@ + + + + + + + content + tez-env template + This is the freemarker template for tez-env.sh file + + + + longtext + + + + enable_heap_dump + false + Enable or disable taking Heap Dump. (true/false) + Enable heap dump + + + heap_dump_location + /tmp + Location for heap dump file + Heap dump location + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml new file mode 100644 index 00000000..98d5f291 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml @@ -0,0 +1,355 @@ + + + + + + tez.lib.uris.classpath + TEZ Lib URIs Classpath + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${tez_home!}/*,${tez_home!}/lib/*,${tez_conf_dir!} + + Comma-delimited list of the location of the Tez libraries Classpath which will be localized for DAGs. + + + + tez.lib.uris + TEZ Lib URIs + ${tez_lib_uris!} + Comma-delimited list of the location of the Tez libraries which will be localized for DAGs. + Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is + uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the + classpath (after . and .*). + If multiple files are specified - files are localized as regular files, contents of directories are + localized as regular files (non-recursive). + + + + tez.cluster.additional.classpath.prefix + /etc/hadoop/conf/secure + + + + tez.am.log.level + INFO + Root Logging level passed to the Tez app master + + + tez.generate.debug.artifacts + false + Generate debug artifacts such as a text representation of the submitted DAG plan + + + tez.staging-dir + TEZ Staging directory + /tmp/${user.name}/staging + The staging dir used while submitting DAGs + + + tez.am.resource.memory.mb + 2048 + The amount of memory to be used by the AppMaster. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.am.launch.cmd-opts + -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!} + Java options for the Tez AppMaster process. The Xmx value is derived based on + tez.am.resource.memory.mb and is 80% of the value by default. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.am.launch.cluster-default.cmd-opts + -server -Djava.net.preferIPv4Stack=true + Cluster default Java options for the Tez AppMaster process. These will be prepended to the + properties specified via tez.am.launch.cmd-opts + + + + tez.am.launch.env + LD_LIBRARY_PATH=${hadoop_home!}/lib/native + + Additional execution environment entries for tez. This is not an additive property. You must preserve the + original value if + you want to have access to native libraries. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.task.resource.memory.mb + 1536 + The amount of memory to be used by launched tasks. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.task.launch.cmd-opts + -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB ${heap_dump_opts!} + Java options for tasks. The Xmx value is derived based on tez.task.resource.memory.mb and is 80% of + this value by default. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.task.launch.cluster-default.cmd-opts + -server -Djava.net.preferIPv4Stack=true + Cluster default Java options for tasks. These will be prepended to the properties specified via + tez.task.launch.cmd-opts + + + + tez.task.launch.env + LD_LIBRARY_PATH=${hadoop_home!}/lib/native + + Additional execution environment entries for tez. This is not an additive property. You must preserve the + original value if + you want to have access to native libraries. + Used only if the value is not specified explicitly by the DAG definition. + + + + tez.shuffle-vertex-manager.min-src-fraction + 0.2 + In case of a ScatterGather connection, the fraction of source tasks which should + complete before tasks for the current vertex are schedule + + + + tez.shuffle-vertex-manager.max-src-fraction + 0.4 + In case of a ScatterGather connection, once this fraction of source tasks have + completed, all tasks on the current vertex can be scheduled. Number of tasks ready for + scheduling on the current vertex scales linearly between min-fraction and max-fraction + + + + tez.am.am-rm.heartbeat.interval-ms.max + 250 + The maximum heartbeat interval between the AM and RM in milliseconds + + + tez.grouping.split-waves + 1.7 + The multiplier for available queue capacity when determining number of tasks for + a Vertex. 1.7 with 100% queue available implies generating a number of tasks roughly equal + to 170% of the available containers on the queue + + + + tez.grouping.min-size + 16777216 + Lower bound on the size (in bytes) of a grouped split, to avoid generating + too many splits + + + + tez.grouping.max-size + 1073741824 + Upper bound on the size (in bytes) of a grouped split, to avoid generating + excessively large split + + + + tez.am.container.reuse.enabled + true + Configuration to specify whether container should be reused + + + tez.am.container.reuse.rack-fallback.enabled + true + Whether to reuse containers for rack local tasks. Active only if reuse is enabled + + + + tez.am.container.reuse.non-local-fallback.enabled + false + Whether to reuse containers for non-local tasks. Active only if reuse is enabled + + + + tez.am.container.idle.release-timeout-min.millis + 10000 + The minimum amount of time to hold on to a container that is idle. Only active when reuse is + enabled. + + + + tez.am.container.idle.release-timeout-max.millis + 20000 + The maximum amount of time to hold on to a container if no task can be assigned to it immediately. + Only active when reuse is enabled. + + + + tez.am.container.reuse.locality.delay-allocation-millis + 250 + The amount of time to wait before assigning a container to the next level of + locality. NODE -> RACK -> NON_LOCAL + + + + tez.am.max.app.attempts + 2 + Specifies the total number of time the app master will run in case recovery is triggered + + + tez.am.maxtaskfailures.per.node + 10 + The maximum number of allowed task attempt failures on a node before it gets marked as blacklisted + + + tez.task.am.heartbeat.counter.interval-ms.max + 4000 + Time interval at which task counters are sent to the AM + + + tez.task.get-task.sleep.interval-ms.max + 200 + The maximum amount of time, in seconds, to wait before a task asks an AM for another task + + + tez.task.max-events-per-heartbeat + 500 + Maximum number of of events to fetch from the AM by the tasks in a single heartbeat. + + + tez.session.client.timeout.secs + -1 + Time (in seconds) to wait for AM to come up when trying to submit a DAG from the client + + + tez.session.am.dag.submit.timeout.secs + 300 + + Time (in seconds) for which the Tez AM should wait for a DAG to be submitted before shutting down + + + + tez.runtime.compress + true + Whether intermediate data should be compressed or not + + + tez.runtime.compress.codec + org.apache.hadoop.io.compress.SnappyCodec + + The coded to be used if compressing intermediate data. Only applicable if tez.runtime.compress is enabled + + + + tez.runtime.unordered.output.buffer.size-mb + 100 + The size of the buffer when output does not require to be sorted + + + tez.runtime.convert.user-payload.to.history-text + false + Whether to publish configuration information to History logger + + + tez.use.cluster.hadoop-libs + false + + This being true implies that the deployment is relying on hadoop jars being available on the cluster on all nodes. + + + + tez.am.tez-ui.history-url.template + __HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__ + Template to generate the History URL for a particular Tez Application. + Template replaces __APPLICATION_ID__ with the actual applicationId and + __HISTORY_URL_BASE__ with the value from the tez.tez-ui.history-url.base config property + + + + tez.tez-ui.history-url.base + + + + + tez.am.view-acls + * + AM view ACLs. This allows the specified users/groups to view the status of the AM and all DAGs that + run within this AM. + Value format: Comma separated list of users, followed by whitespace, followed by a comma separated list of + groups. + + + + tez.runtime.optimize.local.fetch + true + If the shuffle input is on the local host bypass the http fetch and access the files directly. + + + + tez.task.generate.counters.per.io + true + Whether to generate counters on a per-edge basis for a Tez DAG. Helpful for in-depth analysis. + + + + tez.runtime.sorter.class + PIPELINED + Which sorter implementation to use + + + tez.runtime.pipelined.sorter.sort.threads + 2 + Tez runtime pipelined sorter sort threads + + + tez.runtime.io.sort.mb + 272 + The size of the sort buffer when output needs to be sorted + + + tez.counters.max + 10000 + The number of allowed counters for the executing DAG + + + tez.counters.max.groups + 3000 + The number of allowed counter groups for the executing DAG + + + tez.runtime.shuffle.fetch.buffer.percent + 0.6 + Fraction (0-1) of the available memory which can be used to retain shuffled data + + + tez.runtime.shuffle.memory.limit.percent + 0.25 + This property determines the maximum size of a shuffle segment + which can be fetched to memory. Fraction (0-1) of shuffle memory + (after applying tez.runtime.shuffle.fetch.buffer.percent) + + + + tez.history.logging.timeline-cache-plugin.old-num-dags-per-group + 5 + DAGs per group + + + yarn.timeline-service.enabled + false + Timeline service version we’re currently using. + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml new file mode 100644 index 00000000..42ef5e26 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml @@ -0,0 +1,63 @@ + + + + + + tez + Tez + Tez is the next generation Hadoop Query Processing framework written on top of YARN. + 0.10.2-1 + tez + tez + + + + tez_client + Tez Client + client + 1+ + + org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez.TezClientScript + java + 1200 + + + + + + + + centos7 + rocky8 + + + x86_64 + + + tez_3_3_0 + + + + + + hdfs + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml index 996cb2ec..a718fae6 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml @@ -38,8 +38,7 @@ yarn.resourcemanager.address 0.0.0.0:8032 - The address of the applications manager interface in the - RM. + The address of the applications manager interface in the RM. @@ -91,14 +90,12 @@ yarn.nodemanager.resource.memory-mb 5120 - Amount of physical memory, in MB, that can be allocated - for containers. + Amount of physical memory, in MB, that can be allocated for containers. Memory allocated for all YARN containers on a node yarn.application.classpath - - {{hadoop_conf_dir}},{{hadoop_home}}/*,{{hadoop_home}}/lib/*,{{hadoop_hdfs_home}}/*,{{hadoop_hdfs_home}}/lib/*,{{hadoop_yarn_home}}/*,{{hadoop_yarn_home}}/lib/* + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/* Classpath for typical applications. @@ -402,12 +399,12 @@ yarn.authorization-provider - Yarn authorization provider class. + Yarn authorization provider class. yarn.admin.acl yarn - ACL of who can be admin of the YARN cluster. + ACL of who can be admin of the YARN cluster. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml index 44ea8ffe..9e0d03d1 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 yarn YARN diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json index 9c9addb5..ea235ec1 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json @@ -1,16 +1,14 @@ { "RESOURCEMANAGER-START": [ - "ZOOKEEPER_SERVER-START" + "ZOOKEEPER_SERVER-START", + "NAMENODE-START", + "DATANODE-START" ], "NODEMANAGER-START": [ "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START" ], - "RESOURCEMANAGER-START": [ - "NAMENODE-START", - "DATANODE-START" - ], "RESOURCEMANAGER-RESTART": [ "NAMENODE-RESTART" ], diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml index 64f10b14..043b67e4 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 zookeeper ZooKeeper diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json index a0d75ea4..0da47175 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/order.json @@ -1,5 +1,5 @@ { - "KAFKA_BROKER-INSTALL": [ - "ZOOKEEPER_SERVER-INSTALL" + "ZOOKEEPER_SERVER-STOP": [ + "KAFKA_BROKER-STOP" ] } \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml index 1c956f04..c87d7b7d 100644 --- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 nop 1.0.0 diff --git a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml index cd3d23ed..dd3d0c2c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/kafka/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 kafka Kafka diff --git a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml index 55db55e8..1ebb84cd 100644 --- a/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/nop/1.0.0/services/zookeeper/metainfo.xml @@ -19,7 +19,6 @@ --> - 2.0 zookeeper ZooKeeper diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml index ad85d1ae..2e2bbe6b 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml @@ -29,6 +29,11 @@ ${project.artifactId} Bigtop Manager Stack Bigtop + + + 3.3.6 + + org.apache.bigtop @@ -39,6 +44,78 @@ com.google.auto.service auto-service + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java new file mode 100644 index 00000000..fe65590f --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.utils; + +import org.apache.bigtop.manager.stack.common.exception.StackException; +import org.apache.bigtop.manager.stack.common.utils.LocalSettings; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.net.URI; +import java.security.PrivilegedAction; +import java.text.MessageFormat; +import java.util.List; + +@Data +@Slf4j +public class HdfsUtil { + public static void createDirectory(String user, String directory) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + try { + ugi.doAs((PrivilegedAction) () -> { + try (FileSystem fs = getFileSystem()) { + // Create dest dir if not exist + Path destDirPath = new Path(directory); + if (!fs.exists(destDirPath)) { + fs.mkdirs(destDirPath); + } + } catch (Exception e) { + log.error("Error while creating directory on hdfs", e); + throw new StackException(e); + } + + return null; + }); + } catch (Exception e) { + log.error("Error while creating directory on hdfs", e); + throw new StackException(e); + } + } + + public static void uploadFile(String user, String localFilePath, String destDir) { + uploadFile(user, localFilePath, destDir, null); + } + + public static void uploadFile(String user, String localFilePath, String destDir, String destFilename) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + try { + ugi.doAs((PrivilegedAction) () -> { + try (FileSystem fs = getFileSystem()) { + // Create dest dir if not exist + Path destDirPath = new Path(destDir); + if (!fs.exists(destDirPath)) { + fs.mkdirs(destDirPath); + } + + // upload file + Path destFilePath = destFilename == null ? new Path(destDir) : new Path(destDir, destFilename); + fs.copyFromLocalFile(new Path(localFilePath), destFilePath); + } catch (Exception e) { + log.error("Error while uploading file to hdfs", e); + throw new StackException(e); + } + + return null; + }); + } catch (Exception e) { + log.error("Error while uploading file to hdfs", e); + throw new StackException(e); + } + } + + private static FileSystem getFileSystem() throws Exception { + Configuration conf = new Configuration(); + conf.addResource(new Path("/etc/hadoop/conf/core-site.xml")); + conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml")); + + List namenodeList = LocalSettings.hosts("namenode"); + if (CollectionUtils.isEmpty(namenodeList)) { + String msg = "No namenode found in the cluster"; + log.error(msg); + throw new StackException(msg); + } + + String hdfsUri = MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0)); + return FileSystem.get(new URI(hdfsUri), conf); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java index 274eac18..3d37bdf5 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java @@ -83,7 +83,7 @@ public static ShellResult config(Params params, String componentName) { LinuxFileUtils.createDirectories( hdfsParams.getDfsDataDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); LinuxFileUtils.createDirectories( - hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); + hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_775, true); LinuxFileUtils.createDirectories( hdfsParams.getHadoopPidDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); @@ -150,6 +150,10 @@ public static ShellResult config(Params params, String componentName) { Constants.PERMISSION_644, hdfsParams.getGlobalParamsMap()); + // log.info("Creating /apps on hdfs"); + // HdfsUtil.createDirectory(hdfsUser, "/apps"); + + log.info("Successfully configured HDFS"); return ShellResult.success("HDFS Configure success!"); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java index e72467d0..c44c85c6 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java @@ -21,11 +21,8 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.spi.stack.Params; import org.apache.bigtop.manager.spi.stack.Script; -import org.apache.bigtop.manager.stack.common.enums.ConfigType; import org.apache.bigtop.manager.stack.common.exception.StackException; -import org.apache.bigtop.manager.stack.common.utils.LocalSettings; import org.apache.bigtop.manager.stack.common.utils.PackageUtils; -import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils; import org.apache.bigtop.manager.stack.common.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -33,13 +30,6 @@ import java.io.IOException; import java.text.MessageFormat; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644; -import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; -import static org.apache.bigtop.manager.common.constants.Constants.ROOT_USER; @Slf4j @AutoService(Script.class) @@ -52,58 +42,7 @@ public ShellResult install(Params params) { @Override public ShellResult configure(Params params) { - KafkaParams kafkaParams = (KafkaParams) params; - - String confDir = kafkaParams.confDir(); - String kafkaUser = kafkaParams.user(); - String kafkaGroup = kafkaParams.group(); - - LinuxFileUtils.createDirectories(kafkaParams.getKafkaDataDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - LinuxFileUtils.createDirectories(kafkaParams.getKafkaLogDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - LinuxFileUtils.createDirectories(kafkaParams.getKafkaPidDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); - - // server.properties - List zookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); - Map paramMap = new HashMap<>(); - paramMap.put("zk_server_list", zookeeperServerHosts); - paramMap.put("host", kafkaParams.hostname()); - LinuxFileUtils.toFile( - ConfigType.PROPERTIES, - MessageFormat.format("{0}/server.properties", confDir), - kafkaUser, - kafkaGroup, - PERMISSION_644, - kafkaParams.kafkaBroker(), - paramMap); - - // kafka-env - LinuxFileUtils.toFileByTemplate( - kafkaParams.getKafkaEnvContent(), - MessageFormat.format("{0}/kafka-env.sh", confDir), - kafkaUser, - kafkaGroup, - PERMISSION_644, - kafkaParams.getGlobalParamsMap()); - - // log4j - LinuxFileUtils.toFileByTemplate( - kafkaParams.getKafkaLog4jContent(), - MessageFormat.format("{0}/log4j.properties", confDir), - kafkaUser, - kafkaGroup, - PERMISSION_644, - kafkaParams.getGlobalParamsMap()); - - // kafka.limits - LinuxFileUtils.toFileByTemplate( - kafkaParams.kafkaLimits(), - MessageFormat.format("{0}/kafka.conf", KafkaParams.LIMITS_CONF_DIR), - ROOT_USER, - ROOT_USER, - PERMISSION_644, - kafkaParams.getGlobalParamsMap()); - - return ShellResult.success("Kafka Server Configure success!"); + return KafkaSetup.config(params); } @Override diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java new file mode 100644 index 00000000..ca5c2369 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.spi.stack.Params; +import org.apache.bigtop.manager.stack.common.enums.ConfigType; +import org.apache.bigtop.manager.stack.common.utils.LocalSettings; +import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644; +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; +import static org.apache.bigtop.manager.common.constants.Constants.ROOT_USER; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class KafkaSetup { + + public static ShellResult config(Params params) { + KafkaParams kafkaParams = (KafkaParams) params; + + String confDir = kafkaParams.confDir(); + String kafkaUser = kafkaParams.user(); + String kafkaGroup = kafkaParams.group(); + + LinuxFileUtils.createDirectories(kafkaParams.getKafkaDataDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); + LinuxFileUtils.createDirectories(kafkaParams.getKafkaLogDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); + LinuxFileUtils.createDirectories(kafkaParams.getKafkaPidDir(), kafkaUser, kafkaGroup, PERMISSION_755, true); + + // server.properties + List zookeeperServerHosts = LocalSettings.hosts("zookeeper_server"); + Map paramMap = new HashMap<>(); + paramMap.put("zk_server_list", zookeeperServerHosts); + paramMap.put("host", kafkaParams.hostname()); + LinuxFileUtils.toFile( + ConfigType.PROPERTIES, + MessageFormat.format("{0}/server.properties", confDir), + kafkaUser, + kafkaGroup, + PERMISSION_644, + kafkaParams.kafkaBroker(), + paramMap); + + // kafka-env + LinuxFileUtils.toFileByTemplate( + kafkaParams.getKafkaEnvContent(), + MessageFormat.format("{0}/kafka-env.sh", confDir), + kafkaUser, + kafkaGroup, + PERMISSION_644, + kafkaParams.getGlobalParamsMap()); + + // log4j + LinuxFileUtils.toFileByTemplate( + kafkaParams.getKafkaLog4jContent(), + MessageFormat.format("{0}/log4j.properties", confDir), + kafkaUser, + kafkaGroup, + PERMISSION_644, + kafkaParams.getGlobalParamsMap()); + + // kafka.limits + LinuxFileUtils.toFileByTemplate( + kafkaParams.kafkaLimits(), + MessageFormat.format("{0}/kafka.conf", KafkaParams.LIMITS_CONF_DIR), + ROOT_USER, + ROOT_USER, + PERMISSION_644, + kafkaParams.getGlobalParamsMap()); + + return ShellResult.success("Kafka Server Configure success!"); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java new file mode 100644 index 00000000..648680cf --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.spi.stack.ClientScript; +import org.apache.bigtop.manager.spi.stack.Params; +import org.apache.bigtop.manager.spi.stack.Script; +import org.apache.bigtop.manager.stack.common.utils.PackageUtils; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(Script.class) +public class TezClientScript implements ClientScript { + + @Override + public ShellResult install(Params params) { + return PackageUtils.install(params.getPackageList()); + } + + @Override + public ShellResult configure(Params params) { + return TezSetup.config(params); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java new file mode 100644 index 00000000..f5df822e --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.common.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.common.utils.BaseParams; +import org.apache.bigtop.manager.stack.common.utils.LocalSettings; + +import org.apache.commons.lang3.StringUtils; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.Map; + +@Getter +@Slf4j +public class TezParams extends BaseParams { + + private String headDumpOpts; + private String tezEnvContent; + + public TezParams(CommandPayload commandPayload) { + super(commandPayload); + globalParamsMap.put("tez_user", user()); + globalParamsMap.put("tez_group", group()); + globalParamsMap.put("java_home", "/usr/local/java"); + globalParamsMap.put("hadoop_home", hadoopHome()); + globalParamsMap.put("hadoop_conf_dir", hadoopConfDir()); + globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); + globalParamsMap.put("hadoop_yarn_home", yarnHome()); + globalParamsMap.put("tez_home", serviceHome()); + globalParamsMap.put("tez_conf_dir", confDir()); + globalParamsMap.put("head_dump_opts", headDumpOpts); + } + + @GlobalParams + public Map tezSite() { + return LocalSettings.configurations(serviceName(), "tez-site"); + } + + @GlobalParams + public Map tezEnv() { + Map tezEnv = LocalSettings.configurations(serviceName(), "tez-env"); + + String heapDumpEnabled = (String) tezEnv.get("enable_heap_dump"); + if (StringUtils.isNotBlank(heapDumpEnabled) && Boolean.parseBoolean(heapDumpEnabled)) { + String heapDumpLocation = StringUtils.defaultIfBlank((String) tezEnv.get("heap_dump_location"), "/tmp"); + headDumpOpts = + MessageFormat.format("-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={0}", heapDumpLocation); + } + + tezEnvContent = (String) tezEnv.get("content"); + + return tezEnv; + } + + @Override + public String confDir() { + return "/etc/tez/conf"; + } + + @Override + public String serviceHome() { + return stackLibDir() + "/tez"; + } + + public String hadoopConfDir() { + return "/etc/hadoop/conf"; + } + + public String hadoopHome() { + return stackLibDir() + "/hadoop"; + } + + public String hdfsHome() { + return stackLibDir() + "/hadoop-hdfs"; + } + + public String yarnHome() { + return stackLibDir() + "/hadoop-yarn"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java new file mode 100644 index 00000000..68cc1faf --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.spi.stack.Params; +import org.apache.bigtop.manager.stack.common.enums.ConfigType; +import org.apache.bigtop.manager.stack.common.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; + +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class TezSetup { + + public static ShellResult config(Params params) { + TezParams tezParams = (TezParams) params; + + String confDir = tezParams.confDir(); + String tezUser = tezParams.user(); + String tezGroup = tezParams.group(); + + // tez-site + log.info("Generating {}/tez-site.xml file", confDir); + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/tez-site.xml", confDir), + tezUser, + tezGroup, + Constants.PERMISSION_644, + tezParams.tezSite(), + tezParams.getGlobalParamsMap()); + + // tez-env + log.info("Generating {}/tez-env.sh file", confDir); + LinuxFileUtils.toFileByTemplate( + tezParams.getTezEnvContent(), + MessageFormat.format("{0}/tez-env.sh", confDir), + tezUser, + tezGroup, + PERMISSION_755, + tezParams.getGlobalParamsMap()); + + // maybe we should upload tez.tar.gz to HDFS here? + // log.info("Uploading tez.tar.gz to HDFS"); + // HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/tez.tar.gz", "/apps/tez"); + + log.info("Successfully configured Tez"); + return ShellResult.success("Tez Configure success!"); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java index f11c908d..24e35b2d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java @@ -103,7 +103,8 @@ public static ShellResult config(Params params, String componentName) { yarnUser, yarnGroup, Constants.PERMISSION_644, - yarnParams.yarnSite()); + yarnParams.yarnSite(), + yarnParams.getGlobalParamsMap()); // log4j LinuxFileUtils.toFileByTemplate( diff --git a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java index a84e8c65..dff623c7 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/linux/LinuxFileUtils.java @@ -22,6 +22,7 @@ import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.common.utils.YamlUtils; import org.apache.bigtop.manager.stack.common.enums.ConfigType; +import org.apache.bigtop.manager.stack.common.exception.StackException; import org.apache.bigtop.manager.stack.common.utils.template.TemplateUtils; import org.apache.commons.lang3.StringUtils; @@ -226,7 +227,7 @@ public static void createDirectories( Path path = Paths.get(dirPath); if (Files.isSymbolicLink(path)) { - log.warn("unable to create symbolic link: [{}]", dirPath); + log.error("Directory is symbolic link: [{}]", dirPath); return; } @@ -240,4 +241,40 @@ public static void createDirectories( updateOwner(dirPath, owner, group, recursive); updatePermissions(dirPath, permissions, recursive); } + + /** + * create symbolic link + * + * @param target target file + * @param source source file + */ + public static void createSymbolicLink(String target, String source) { + if (StringUtils.isBlank(target) || StringUtils.isBlank(source)) { + log.error("target, source must not be null"); + return; + } + + Path targetPath = Paths.get(target); + Path sourcePath = Paths.get(source); + + try { + log.info("Creating symbolic link: [{}] -> [{}]", target, source); + if (Files.exists(targetPath)) { + if (Files.isSymbolicLink(targetPath)) { + Path existingSourcePath = Files.readSymbolicLink(targetPath); + if (!existingSourcePath.equals(sourcePath)) { + throw new IOException( + "Symbolic link already exists and points to a different source: " + existingSourcePath); + } + } else { + throw new IOException("Target path exists and is not a symbolic link: " + target); + } + } + + Files.createSymbolicLink(targetPath, sourcePath); + } catch (IOException e) { + log.error("Error when create symbolic link", e); + throw new StackException(e); + } + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java index c07b5ec8..a6b2b67f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java @@ -58,7 +58,7 @@ private void addUserAndGroup() { } log.info( - "Adding user: {} to group: {} and groups: [{}]", + "Adding user: {} to primary group: {} and supplementary groups: [{}]", user.getKey(), userGroup, String.join(",", groups)); diff --git a/bigtop-manager-stack/pom.xml b/bigtop-manager-stack/pom.xml index 98556bb2..c9bf8637 100644 --- a/bigtop-manager-stack/pom.xml +++ b/bigtop-manager-stack/pom.xml @@ -52,6 +52,7 @@ pom import + com.google.auto.service auto-service diff --git a/dev-support/docker/rocky8/build-containers.sh b/dev-support/docker/rocky8/build-containers.sh index fda727a8..effd34f2 100755 --- a/dev-support/docker/rocky8/build-containers.sh +++ b/dev-support/docker/rocky8/build-containers.sh @@ -50,7 +50,7 @@ echo -e "\033[32mCreating network bigtop-manager\033[0m" docker network create --driver bridge bigtop-manager echo -e "\033[32mCreating container bigtop-manager-server\033[0m" -docker run -it -d -p 13306:3306 -p 15005:5005 -p 18080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 +docker run -it -d -p 13306:3306 -p 15005:5005 -p 15006:5006 -p 18080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 docker cp ../../../bigtop-manager-server/target/bigtop-manager-server bigtop-manager-server:/opt/ docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-server:/opt/ SERVER_PUB_KEY=`docker exec bigtop-manager-server /bin/cat /root/.ssh/id_rsa.pub` @@ -116,7 +116,7 @@ docker exec bigtop-manager-agent-01 bash -c "sed -i 's/host: localhost/host: $BI docker exec bigtop-manager-agent-02 bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" +docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &" docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &"