From f74e9d29932343a567bf2c05aefe8231c2c761dc Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Mon, 23 Dec 2024 17:22:40 +0800 Subject: [PATCH] BIGTOP-4134: Add Spark component on Bigtop-3.3.0 stack --- .../bigtop/3.3.0/services/hive/order.json | 4 + .../spark/configuration/fairscheduler.xml | 65 +++++ .../services/spark/configuration/log4j2.xml | 103 ++++++++ .../services/spark/configuration/metrics.xml | 244 ++++++++++++++++++ .../spark/configuration/spark-defaults.xml | 66 +++++ .../spark/configuration/spark-env.xml | 91 +++++++ .../spark/configuration/spark-hive-site.xml | 28 ++ .../bigtop/3.3.0/services/spark/metainfo.xml | 80 ++++++ .../bigtop/3.3.0/services/spark/order.json | 14 + .../v3_3_0/spark/SparkClientScript.java | 52 ++++ .../spark/SparkHistoryServerScript.java | 85 ++++++ .../bigtop/v3_3_0/spark/SparkParams.java | 134 ++++++++++ .../stack/bigtop/v3_3_0/spark/SparkSetup.java | 107 ++++++++ .../v3_3_0/spark/SparkThriftServerScript.java | 85 ++++++ .../core/utils/linux/LinuxFileUtils.java | 1 + 15 files changed, 1159 insertions(+) create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/fairscheduler.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/log4j2.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/metrics.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-defaults.xml create mode 100755 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-env.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-hive-site.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/metainfo.xml create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/order.json create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkParams.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkSetup.java create mode 100644 bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json index 19452cf0..e5b220ce 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/order.json @@ -9,6 +9,10 @@ "NAMENODE-RESTART", "NODEMANAGER-RESTART" ], + "HIVE_METASTORE-STOP": [ + "SPARK_HISTORYSERVER-STOP", + "SPARK_THRIFTSERVER-STOP" + ], "HIVESERVER2-START": [ "NODEMANAGER-START", "ZOOKEEPER_SERVER-START", diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/fairscheduler.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/fairscheduler.xml new file mode 100644 index 00000000..32463131 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/fairscheduler.xml @@ -0,0 +1,65 @@ + + + + + + content + fairscheduler template + Custom fairscheduler.xml + + + +<#noparse> + + + FAIR + 1 + 2 + + + FIFO + 2 + 3 + + + +]]> + + + longtext + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/log4j2.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/log4j2.xml new file mode 100644 index 00000000..ab704a43 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/log4j2.xml @@ -0,0 +1,103 @@ + + + + + + content + log4j2 template + Custom log4j2.properties + +# Set everything to be logged to the console +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = console + +# In the pattern layout configuration below, we specify an explicit `%ex` conversion +# pattern for logging Throwables. If this was omitted, then (by default) Log4J would +# implicitly add an `%xEx` conversion pattern which logs stacktraces with additional +# class packaging information. That extra information can sometimes add a substantial +# performance overhead, so we disable it in our default logging config. +# For more information, see SPARK-39361. +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex + +# Set the default spark-shell/spark-sql log level to WARN. When running the +# spark-shell/spark-sql, the log level for these classes is used to overwrite +# the root logger's log level, so that the user can have different defaults +# for the shell and regular Spark apps. +logger.repl.name = org.apache.spark.repl.Main +logger.repl.level = warn + +logger.thriftserver.name = org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver +logger.thriftserver.level = warn + +# Settings to quiet third party logs that are too verbose +logger.jetty1.name = org.sparkproject.jetty +logger.jetty1.level = warn +logger.jetty2.name = org.sparkproject.jetty.util.component.AbstractLifeCycle +logger.jetty2.level = error +logger.replexprTyper.name = org.apache.spark.repl.SparkIMain$exprTyper +logger.replexprTyper.level = info +logger.replSparkILoopInterpreter.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter +logger.replSparkILoopInterpreter.level = info +logger.parquet1.name = org.apache.parquet +logger.parquet1.level = error +logger.parquet2.name = parquet +logger.parquet2.level = error + +# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support +logger.RetryingHMSHandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler +logger.RetryingHMSHandler.level = fatal +logger.FunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry +logger.FunctionRegistry.level = error + +# For deploying Spark ThriftServer +# SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 +appender.console.filter.1.type = RegexFilter +appender.console.filter.1.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.onMatch = deny +appender.console.filter.1.onMismatch = neutral + +]]> + + + longtext + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/metrics.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/metrics.xml new file mode 100644 index 00000000..ed692405 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/metrics.xml @@ -0,0 +1,244 @@ + + + + + + content + metrics template + Custom metrics.properties + +## Examples +# Enable JmxSink for all instances by class name +#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink + +# Enable ConsoleSink for all instances by class name +#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink + +# Enable StatsdSink for all instances by class name +#*.sink.statsd.class=org.apache.spark.metrics.sink.StatsdSink +#*.sink.statsd.prefix=spark + +# Polling period for the ConsoleSink +#*.sink.console.period=10 +# Unit of the polling period for the ConsoleSink +#*.sink.console.unit=seconds + +# Polling period for the ConsoleSink specific for the master instance +#master.sink.console.period=15 +# Unit of the polling period for the ConsoleSink specific for the master +# instance +#master.sink.console.unit=seconds + +# Enable CsvSink for all instances by class name +#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink + +# Polling period for the CsvSink +#*.sink.csv.period=1 +# Unit of the polling period for the CsvSink +#*.sink.csv.unit=minutes + +# Polling directory for CsvSink +#*.sink.csv.directory=/tmp/ + +# Polling period for the CsvSink specific for the worker instance +#worker.sink.csv.period=10 +# Unit of the polling period for the CsvSink specific for the worker instance +#worker.sink.csv.unit=minutes + +# Enable Slf4jSink for all instances by class name +#*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink + +# Polling period for the Slf4JSink +#*.sink.slf4j.period=1 +# Unit of the polling period for the Slf4jSink +#*.sink.slf4j.unit=minutes + +# Example configuration for Graphite sink +#*.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink +#*.sink.graphite.host= +#*.sink.graphite.port= +#*.sink.graphite.period=10 +#*.sink.graphite.unit=seconds +#*.sink.graphite.prefix= + +# Enable JvmSource for instance master, worker, driver and executor +#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource + +#worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource + +#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource + +#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource + +# Example configuration for PrometheusServlet +#*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet +#*.sink.prometheusServlet.path=/metrics/prometheus +#master.sink.prometheusServlet.path=/metrics/master/prometheus +#applications.sink.prometheusServlet.path=/metrics/applications/prometheus + +]]> + + + longtext + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-defaults.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-defaults.xml new file mode 100644 index 00000000..aca92442 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-defaults.xml @@ -0,0 +1,66 @@ + + + + + + spark_history_fs_logDirectory + file:/tmp/spark-events + + + content + spark-defaults template + Custom spark-defaults.conf + + + + longtext + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-env.xml new file mode 100755 index 00000000..9d21863b --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-env.xml @@ -0,0 +1,91 @@ + + + + + + spark_log_dir + Spark Log directory + /var/log/spark + Spark Log Dir + + + spark_pid_dir + Spark PID directory + /var/run/spark + + + + content + This is the freemarker template for spark-env.sh file + +export SPARK_DIST_CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath) + + +# The java implementation to use. +export JAVA_HOME=${java_home} + +# Add hadoop native libraries support for better performance +export LD_LIBRARY_PATH=${hadoop_home}/lib/native/:$LD_LIBRARY_PATH +]]> + + + longtext + + + diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-hive-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-hive-site.xml new file mode 100644 index 00000000..57699342 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/configuration/spark-hive-site.xml @@ -0,0 +1,28 @@ + + + + + + hive.server2.thrift.port + 10015 + Spark ThriftServer Port + TCP port number to listen on, default 10015. + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/metainfo.xml new file mode 100644 index 00000000..63a3382e --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/metainfo.xml @@ -0,0 +1,80 @@ + + + + + + spark + Spark + + Apache Spark is a unified analytics engine for large-scale data processing. + + 3.3.4-1 + spark + + + + spark_historyserver + Spark HistoryServer + server + 1+ + + Spark HistoryServer UI + spark.history.ui.port + 18080 + spark.history.ui.port + 18080 + + + + + spark_thriftserver + Spark ThriftServer + server + 1+ + + + + spark_client + Spark Client + client + 1+ + + + + + + + x86_64 + aarch64 + + + + spark-3.3.4-1.tar.gz + SHA-256:91ff4fa884abee1156d7819d16bf3035e98c98833237cfd457b2d0b1b8ad42e5 + + + + + + + hive + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/order.json new file mode 100644 index 00000000..a26d2b3e --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/spark/order.json @@ -0,0 +1,14 @@ +{ + "SPARK_HISTORYSERVER-START": [ + "HIVE_METASTORE-START" + ], + "SPARK_HISTORYSERVER-RESTART": [ + "HIVE_METASTORE-RESTART" + ], + "SPARK_THRIFTSERVER-START": [ + "HIVE_METASTORE-START" + ], + "SPARK_THRIFTSERVER-RESTART": [ + "HIVE_METASTORE-RESTART" + ] +} \ No newline at end of file diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java new file mode 100644 index 00000000..6b1df772 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.spark; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; +import org.apache.bigtop.manager.stack.core.spi.script.Script; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.util.Properties; + +@Slf4j +@AutoService(Script.class) +public class SparkClientScript extends AbstractClientScript { + + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + + @Override + public ShellResult configure(Params params) { + return SparkSetup.configure(params); + } + + @Override + public String getComponentName() { + return "spark_client"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java new file mode 100644 index 00000000..32beacec --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.spark; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; +import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.Properties; + +@Slf4j +@AutoService(Script.class) +public class SparkHistoryServerScript extends AbstractServerScript { + + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + + @Override + public ShellResult configure(Params params) { + return SparkSetup.configure(params); + } + + @Override + public ShellResult start(Params params) { + configure(params); + SparkParams sparkParams = (SparkParams) params; + String cmd = MessageFormat.format("{0}/sbin/start-history-server.sh", sparkParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, sparkParams.user()); + } catch (IOException e) { + throw new StackException(e); + } + } + + @Override + public ShellResult stop(Params params) { + SparkParams sparkParams = (SparkParams) params; + String cmd = MessageFormat.format("{0}/sbin/stop-history-server.sh", sparkParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, sparkParams.user()); + } catch (IOException e) { + throw new StackException(e); + } + } + + @Override + public ShellResult status(Params params) { + SparkParams sparkParams = (SparkParams) params; + return LinuxOSUtils.checkProcess(sparkParams.getSparkHistoryServerPidFile()); + } + + @Override + public String getComponentName() { + return "spark_historyserver"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkParams.java new file mode 100644 index 00000000..dd733204 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkParams.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.spark; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; +import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.utils.LocalSettings; + +import com.google.auto.service.AutoService; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Getter +@AutoService(Params.class) +@NoArgsConstructor +public class SparkParams extends BigtopParams { + + private String sparkLogDir = "/var/log/spark"; + private String sparkPidDir = "/var/run/spark"; + private String sparkHistoryLogDir; + private String sparkHistoryServerPidFile; + private String sparkThriftServerPidFile; + + private String sparkEnvContent; + private String sparkLog4j2Content; + private String sparkMetricsContent; + private String sparkFairSchedulerContent; + private String sparkDefaultsContent; + + public SparkParams(CommandPayload commandPayload) { + super(commandPayload); + globalParamsMap.put("spark_user", user()); + globalParamsMap.put("spark_group", group()); + globalParamsMap.put("java_home", javaHome()); + globalParamsMap.put("spark_conf_dir", confDir()); + globalParamsMap.put("hadoop_home", hadoopHome()); + globalParamsMap.put("hadoop_conf_dir", hadoopConfDir()); + globalParamsMap.put("hive_home", hiveHome()); + globalParamsMap.put("hive_conf_dir", hiveConfDir()); + globalParamsMap.put("security_enabled", false); + } + + @GlobalParams + public Map sparkFairScheduler() { + Map sparkFairScheduler = LocalSettings.configurations(getServiceName(), "fairscheduler"); + sparkFairSchedulerContent = (String) sparkFairScheduler.get("content"); + return sparkFairScheduler; + } + + @GlobalParams + public Map sparkMetrics() { + Map sparkMetrics = LocalSettings.configurations(getServiceName(), "metrics"); + sparkMetricsContent = (String) sparkMetrics.get("content"); + return sparkMetrics; + } + + @GlobalParams + public Map sparkDefaults() { + Map sparkDefaults = LocalSettings.configurations(getServiceName(), "spark-defaults"); + sparkHistoryLogDir = sparkDefaults.get("spark_history_fs_logDirectory").toString(); + sparkDefaultsContent = (String) sparkDefaults.get("content"); + return sparkDefaults; + } + + @GlobalParams + public Map sparkEnv() { + Map sparkEnv = LocalSettings.configurations(getServiceName(), "spark-env"); + sparkPidDir = sparkEnv.get("spark_pid_dir").toString(); + sparkHistoryServerPidFile = + sparkPidDir + "/spark-" + user() + "-org.apache.spark.deploy.history.HistoryServer-1.pid"; + sparkThriftServerPidFile = + sparkPidDir + "/spark-" + user() + "-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid"; + sparkLogDir = sparkEnv.get("spark_log_dir").toString(); + sparkEnvContent = sparkEnv.get("content").toString(); + return sparkEnv; + } + + @GlobalParams + public Map sparkHiveSite() { + Map configurations = LocalSettings.configurations(getServiceName(), "spark-hive-site"); + Map hiveSite = LocalSettings.configurations("hive", "hive-site"); + configurations.put("hive.metastore.uris", hiveSite.get("hive.metastore.uris")); + configurations.put("hive.metastore.warehouse.dir", hiveSite.get("hive.metastore.warehouse.dir")); + return configurations; + } + + @GlobalParams + public Map sparkLog4j2() { + Map sparkLog4j2 = LocalSettings.configurations(getServiceName(), "log4j2"); + sparkLog4j2Content = (String) sparkLog4j2.get("content"); + return sparkLog4j2; + } + + public String hadoopConfDir() { + return hadoopHome() + "/etc/hadoop"; + } + + public String hadoopHome() { + return stackHome() + "/hadoop"; + } + + public String hiveConfDir() { + return hiveHome() + "/conf"; + } + + public String hiveHome() { + return stackHome() + "/hive"; + } + + @Override + public String getServiceName() { + return "spark"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkSetup.java new file mode 100644 index 00000000..9eb421ff --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkSetup.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.spark; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.enums.ConfigType; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; + +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class SparkSetup { + + public static ShellResult configure(Params params) { + log.info("Configuring Spark"); + SparkParams sparkParams = (SparkParams) params; + + String confDir = sparkParams.confDir(); + String sparkUser = sparkParams.user(); + String sparkGroup = sparkParams.group(); + + LinuxFileUtils.createDirectories(sparkParams.getSparkLogDir(), sparkUser, sparkGroup, PERMISSION_755, true); + LinuxFileUtils.createDirectories(sparkParams.getSparkPidDir(), sparkUser, sparkGroup, PERMISSION_755, true); + + String sparkHistoryLogDir = sparkParams.getSparkHistoryLogDir(); + if (sparkHistoryLogDir.startsWith("file:")) { + String dir = sparkHistoryLogDir.split(":")[1]; + LinuxFileUtils.createDirectories(dir, sparkUser, sparkGroup, PERMISSION_755, true); + } + + LinuxFileUtils.toFileByTemplate( + sparkParams.getSparkEnvContent(), + MessageFormat.format("{0}/spark-env.sh", confDir), + sparkUser, + sparkGroup, + Constants.PERMISSION_755, + sparkParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + sparkParams.getSparkDefaultsContent(), + MessageFormat.format("{0}/spark-defaults.conf", confDir), + sparkUser, + sparkGroup, + Constants.PERMISSION_755, + sparkParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + sparkParams.getSparkFairSchedulerContent(), + MessageFormat.format("{0}/fairscheduler.xml", confDir), + sparkUser, + sparkGroup, + Constants.PERMISSION_755, + sparkParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + sparkParams.getSparkLog4j2Content(), + MessageFormat.format("{0}/log4j2.properties", confDir), + sparkUser, + sparkGroup, + Constants.PERMISSION_755, + sparkParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + sparkParams.getSparkMetricsContent(), + MessageFormat.format("{0}/metrics.properties", confDir), + sparkUser, + sparkGroup, + Constants.PERMISSION_755, + sparkParams.getGlobalParamsMap()); + + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/hive-site.xml", confDir), + sparkUser, + sparkGroup, + Constants.PERMISSION_644, + sparkParams.sparkHiveSite()); + + log.info("Successfully configured Spark"); + return ShellResult.success(); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java new file mode 100644 index 00000000..df9e9f09 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.spark; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; +import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.Properties; + +@Slf4j +@AutoService(Script.class) +public class SparkThriftServerScript extends AbstractServerScript { + + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + + @Override + public ShellResult configure(Params params) { + return SparkSetup.configure(params); + } + + @Override + public ShellResult start(Params params) { + configure(params); + SparkParams sparkParams = (SparkParams) params; + String cmd = MessageFormat.format("{0}/sbin/start-thriftserver.sh", sparkParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, sparkParams.user()); + } catch (IOException e) { + throw new StackException(e); + } + } + + @Override + public ShellResult stop(Params params) { + SparkParams sparkParams = (SparkParams) params; + String cmd = MessageFormat.format("{0}/sbin/stop-thriftserver.sh", sparkParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, sparkParams.user()); + } catch (IOException e) { + throw new StackException(e); + } + } + + @Override + public ShellResult status(Params params) { + SparkParams sparkParams = (SparkParams) params; + return LinuxOSUtils.checkProcess(sparkParams.getSparkHistoryServerPidFile()); + } + + @Override + public String getComponentName() { + return "spark_thriftserver"; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java index 87070926..39474a6a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java @@ -214,6 +214,7 @@ public static void copyFile(String source, String dest) { return; } + log.info("Copy file: [{}] to [{}]", source, dest); List builderParameters = new ArrayList<>(); builderParameters.add("cp"); if (Files.exists(Path.of(source)) && Files.isDirectory(Paths.get(source))) {