From 26c88974782d103b94488b528bf60825f65ad3d1 Mon Sep 17 00:00:00 2001 From: Zhiguo Wu Date: Thu, 23 Jan 2025 18:05:58 +0800 Subject: [PATCH] BIGTOP-4332: Add templates to service metainfo indicates user unmodifiable files (#152) --- .../grpc/payload/ComponentCommandPayload.java | 3 ++ .../manager/grpc/pojo/TemplateInfo.java | 31 +++++++++++++++++ .../command/helper/ComponentStageHelper.java | 6 ++-- .../command/stage/AbstractComponentStage.java | 10 +++--- .../command/stage/ComponentAddStage.java | 4 ++- .../command/stage/ComponentCheckStage.java | 4 ++- .../stage/ComponentConfigureStage.java | 4 ++- .../command/stage/ComponentInitStage.java | 4 ++- .../command/stage/ComponentPrepareStage.java | 4 ++- .../command/stage/ComponentStartStage.java | 4 ++- .../command/stage/ComponentStopStage.java | 4 ++- .../server/command/stage/StageContext.java | 6 ++-- .../command/task/AbstractComponentTask.java | 30 +++++++++++++++-- .../manager/server/model/dto/ServiceDTO.java | 2 ++ .../manager/server/model/dto/TemplateDTO.java | 29 ++++++++++++++++ .../server/stack/model/ServiceModel.java | 4 +++ .../server/stack/model/TemplateModel.java | 33 +++++++++++++++++++ .../manager/server/utils/StackUtils.java | 19 +++++++++++ .../bigtop/3.3.0/services/hive/metainfo.xml | 7 ++++ .../services/hive/template/hive-service.sh | 18 ++++++++++ .../v3_3_0/flink/FlinkClientScript.java | 2 ++ .../flink/FlinkHistoryServerScript.java | 2 ++ .../bigtop/v3_3_0/hadoop/DataNodeScript.java | 2 ++ .../v3_3_0/hadoop/HadoopClientScript.java | 2 ++ .../v3_3_0/hadoop/HistoryServerScript.java | 2 ++ .../bigtop/v3_3_0/hadoop/NameNodeScript.java | 2 ++ .../v3_3_0/hadoop/NodeManagerScript.java | 2 ++ .../v3_3_0/hadoop/ResourceManagerScript.java | 2 ++ .../bigtop/v3_3_0/hadoop/SNameNodeScript.java | 2 ++ .../v3_3_0/hbase/HBaseClientScript.java | 2 ++ .../bigtop/v3_3_0/hbase/HMasterScript.java | 2 ++ .../v3_3_0/hbase/HRegionServerScript.java | 2 ++ .../bigtop/v3_3_0/hive/HiveClientScript.java | 2 ++ .../v3_3_0/hive/HiveMetastoreScript.java | 2 ++ .../stack/bigtop/v3_3_0/hive/HiveParams.java | 3 -- .../bigtop/v3_3_0/hive/HiveServer2Script.java | 2 ++ .../stack/bigtop/v3_3_0/hive/HiveSetup.java | 8 ----- .../v3_3_0/kafka/KafkaBrokerScript.java | 2 ++ .../v3_3_0/solr/SolrInstanceScript.java | 2 ++ .../v3_3_0/spark/SparkClientScript.java | 2 ++ .../spark/SparkHistoryServerScript.java | 2 ++ .../v3_3_0/spark/SparkThriftServerScript.java | 2 ++ .../bigtop/v3_3_0/tez/TezClientScript.java | 2 ++ .../zookeeper/ZookeeperClientScript.java | 2 ++ .../zookeeper/ZookeeperServerScript.java | 2 ++ .../stack/core/spi/param/BaseParams.java | 6 ++++ .../manager/stack/core/spi/param/Params.java | 3 ++ .../stack/core/spi/script/AbstractScript.java | 25 ++++++++++++++ .../seatunnel/SeaTunnelClientScript.java | 2 ++ .../seatunnel/SeaTunnelMasterScript.java | 2 ++ .../seatunnel/SeaTunnelWorkerScript.java | 2 ++ .../v1_0_0/grafana/GrafanaServerScript.java | 2 ++ .../infra/v1_0_0/mysql/MySQLClientScript.java | 2 ++ .../infra/v1_0_0/mysql/MySQLServerScript.java | 2 ++ .../prometheus/PrometheusServerScript.java | 2 ++ 55 files changed, 297 insertions(+), 34 deletions(-) create mode 100644 bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/TemplateInfo.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/TemplateDTO.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/TemplateModel.java create mode 100644 bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/template/hive-service.sh diff --git a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/payload/ComponentCommandPayload.java b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/payload/ComponentCommandPayload.java index a17f768e..f40e0a70 100644 --- a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/payload/ComponentCommandPayload.java +++ b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/payload/ComponentCommandPayload.java @@ -19,6 +19,7 @@ package org.apache.bigtop.manager.grpc.payload; import org.apache.bigtop.manager.grpc.pojo.PackageSpecificInfo; +import org.apache.bigtop.manager.grpc.pojo.TemplateInfo; import lombok.Data; @@ -44,6 +45,8 @@ public class ComponentCommandPayload { private List packageSpecifics; + private List templates; + /** * This field is exclusively used for Prometheus and Grafana within the infra services. * Includes cluster and corresponding hostname. diff --git a/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/TemplateInfo.java b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/TemplateInfo.java new file mode 100644 index 00000000..b7217c0c --- /dev/null +++ b/bigtop-manager-grpc/src/main/java/org/apache/bigtop/manager/grpc/pojo/TemplateInfo.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.grpc.pojo; + +import lombok.Data; + +@Data +public class TemplateInfo { + + private String src; + + private String dest; + + private String content; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java index 05ab40d2..49cddd16 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/helper/ComponentStageHelper.java @@ -31,7 +31,6 @@ import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.exception.ServerException; import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.ComponentDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.utils.StackUtils; @@ -121,11 +120,10 @@ private static StageContext createStageContext( StageContext stageContext = StageContext.fromCommandDTO(commandDTO); ServiceDTO serviceDTO = StackUtils.getServiceDTOByComponentName(componentName); - ComponentDTO componentDTO = StackUtils.getComponentDTO(componentName); stageContext.setHostnames(hostnames); - stageContext.setServiceDTO(serviceDTO); - stageContext.setComponentDTO(componentDTO); + stageContext.setServiceName(serviceDTO.getName()); + stageContext.setComponentName(componentName); return stageContext; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java index 634b8554..0e913467 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java @@ -25,6 +25,7 @@ import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.ComponentDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; +import org.apache.bigtop.manager.server.utils.StackUtils; import java.util.ArrayList; import java.util.HashMap; @@ -55,17 +56,17 @@ protected void beforeCreateTasks() { @Override protected String getServiceName() { - return stageContext.getServiceDTO().getName(); + return stageContext.getServiceName(); } @Override protected String getComponentName() { - return stageContext.getComponentDTO().getName(); + return stageContext.getComponentName(); } protected TaskContext createTaskContext(String hostname) { - ServiceDTO serviceDTO = stageContext.getServiceDTO(); - ComponentDTO componentDTO = stageContext.getComponentDTO(); + ServiceDTO serviceDTO = StackUtils.getServiceDTO(stageContext.getServiceName()); + ComponentDTO componentDTO = StackUtils.getComponentDTO(stageContext.getComponentName()); TaskContext taskContext = new TaskContext(); taskContext.setHostname(hostname); @@ -79,7 +80,6 @@ protected TaskContext createTaskContext(String hostname) { taskContext.setRootDir(clusterPO.getRootDir()); Map properties = new HashMap<>(); - properties.put("packageSpecifics", serviceDTO.getPackageSpecifics()); properties.put("clusterHosts", getClusterHosts()); taskContext.setProperties(properties); return taskContext; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java index 883ce8ef..5c760e18 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentAddStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentAddTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentAddStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Add " + stageContext.getComponentDTO().getDisplayName(); + return "Add " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java index 7498ce7d..ffa2515a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentCheckStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentCheckTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentCheckStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Check " + stageContext.getComponentDTO().getDisplayName(); + return "Check " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java index 08eea2ad..45c213c9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentConfigureStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentConfigureTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentConfigureStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Configure " + stageContext.getComponentDTO().getDisplayName(); + return "Configure " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java index 2cc48652..0a256499 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentInitStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentInitTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentInitStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Init " + stageContext.getComponentDTO().getDisplayName(); + return "Init " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java index 3b91dc8b..b0719bfa 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentPrepareStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentPrepareTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentPrepareStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Prepare " + stageContext.getComponentDTO().getDisplayName(); + return "Prepare " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java index 00f67f8a..bd5cf455 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStartStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentStartTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentStartStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Start " + stageContext.getComponentDTO().getDisplayName(); + return "Start " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java index b3678150..e9ed529d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/ComponentStopStage.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.command.task.ComponentStopTask; import org.apache.bigtop.manager.server.command.task.Task; +import org.apache.bigtop.manager.server.utils.StackUtils; public class ComponentStopStage extends AbstractComponentStage { @@ -34,6 +35,7 @@ protected Task createTask(String hostname) { @Override public String getName() { - return "Stop " + stageContext.getComponentDTO().getDisplayName(); + return "Stop " + + StackUtils.getComponentDTO(stageContext.getComponentName()).getDisplayName(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java index a3823b4c..7bddcaa9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/StageContext.java @@ -22,8 +22,6 @@ import org.apache.bigtop.manager.dao.repository.ClusterDao; import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.ComponentDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import lombok.Data; @@ -43,9 +41,9 @@ public class StageContext { private List hostnames; - private ServiceDTO serviceDTO; + private String serviceName; - private ComponentDTO componentDTO; + private String componentName; public static StageContext fromCommandDTO(CommandDTO commandDTO) { StageContext context = new StageContext(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java index 24ab13c1..9d141fc2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java @@ -27,11 +27,14 @@ import org.apache.bigtop.manager.grpc.payload.ComponentCommandPayload; import org.apache.bigtop.manager.grpc.pojo.PackageInfo; import org.apache.bigtop.manager.grpc.pojo.PackageSpecificInfo; +import org.apache.bigtop.manager.grpc.pojo.TemplateInfo; import org.apache.bigtop.manager.server.grpc.GrpcClient; import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.PackageDTO; import org.apache.bigtop.manager.server.model.dto.PackageSpecificDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; +import org.apache.bigtop.manager.server.model.dto.TemplateDTO; import org.apache.bigtop.manager.server.utils.StackUtils; import java.util.ArrayList; @@ -66,6 +69,8 @@ protected Boolean doRun(String hostname, Integer grpcPort) { @SuppressWarnings("unchecked") protected ComponentCommandRequest getComponentCommandRequest() { StackDTO stackDTO = StackUtils.getServiceStack(taskContext.getServiceName()); + ServiceDTO serviceDTO = StackUtils.getServiceDTO(taskContext.getServiceName()); + ComponentCommandPayload payload = new ComponentCommandPayload(); payload.setServiceName(taskContext.getServiceName()); payload.setComponentName(taskContext.getComponentName()); @@ -75,10 +80,10 @@ protected ComponentCommandRequest getComponentCommandRequest() { payload.setCommand(getCommand().getCode()); payload.setCustomCommand(getCustomCommand()); - Map properties = taskContext.getProperties(); + payload.setTemplates(convertTemplateInfo(serviceDTO.getName(), serviceDTO.getTemplates())); + payload.setPackageSpecifics(convertPackageSpecificInfo(serviceDTO.getPackageSpecifics())); - payload.setPackageSpecifics( - convertPackageSpecificInfo((List) properties.get("packageSpecifics"))); + Map properties = taskContext.getProperties(); if (stackDTO.getStackName().equals("infra")) { Map> clusterHosts = (Map>) properties.get("clusterHosts"); payload.setClusterHosts(clusterHosts); @@ -91,6 +96,25 @@ protected ComponentCommandRequest getComponentCommandRequest() { return requestBuilder.build(); } + private List convertTemplateInfo(String serviceName, List templateDTOList) { + if (templateDTOList == null) { + return new ArrayList<>(); + } + + List templateInfos = new ArrayList<>(); + for (TemplateDTO templateDTO : templateDTOList) { + String content = StackUtils.SERVICE_TEMPLATE_MAP.get(serviceName).get(templateDTO.getSrc()); + + TemplateInfo templateInfo = new TemplateInfo(); + templateInfo.setSrc(templateDTO.getSrc()); + templateInfo.setDest(templateDTO.getDest()); + templateInfo.setContent(content); + templateInfos.add(templateInfo); + } + + return templateInfos; + } + private List convertPackageSpecificInfo(List packageSpecificDTOList) { if (packageSpecificDTOList == null) { return new ArrayList<>(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java index 9b42cb73..25f2c920 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java @@ -37,6 +37,8 @@ public class ServiceDTO { private List packageSpecifics; + private List templates; + private List components; private List configs; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/TemplateDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/TemplateDTO.java new file mode 100644 index 00000000..6a0b96e0 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/TemplateDTO.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.model.dto; + +import lombok.Data; + +@Data +public class TemplateDTO { + + private String src; + + private String dest; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java index 00a1092d..ac17b2ca 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java @@ -50,6 +50,10 @@ public class ServiceModel { @XmlElements(@XmlElement(name = "component")) private List components; + @XmlElementWrapper(name = "templates") + @XmlElements(@XmlElement(name = "template")) + private List templates; + @XmlElementWrapper(name = "required-services") @XmlElements(@XmlElement(name = "service")) private List requiredServices; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/TemplateModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/TemplateModel.java new file mode 100644 index 00000000..1a8a89f9 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/TemplateModel.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.stack.model; + +import lombok.Data; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; + +@Data +@XmlAccessorType(XmlAccessType.FIELD) +public class TemplateModel { + + private String src; + + private String dest; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java index 7eaad4a1..1f49dc47 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/utils/StackUtils.java @@ -21,6 +21,7 @@ import org.apache.bigtop.manager.common.constants.ComponentCategories; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.utils.CaseUtils; +import org.apache.bigtop.manager.common.utils.FileUtils; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.common.utils.ProjectPathUtils; import org.apache.bigtop.manager.server.exception.ApiException; @@ -63,12 +64,16 @@ public class StackUtils { private static final String CONFIGURATION_FOLDER = "configuration"; + private static final String TEMPLATE_FOLDER = "template"; + private static final String CONFIGURATION_FILE_EXTENSION = "xml"; private static final String DEPENDENCY_FILE_NAME = "order.json"; public static final Map> SERVICE_CONFIG_MAP = new HashMap<>(); + public static final Map> SERVICE_TEMPLATE_MAP = new HashMap<>(); + public static final Map> STACK_SERVICE_MAP = new HashMap<>(); public static final DAG DAG = new DAG<>(); @@ -110,6 +115,7 @@ public static void parseService(StackDTO stackDTO, File versionFolder) { services.add(serviceDTO); parseServiceConfigurations(file, serviceDTO.getName()); + parseServiceTemplates(file, serviceDTO.getName()); parseDag(file); } @@ -151,6 +157,19 @@ private static void parseServiceConfigurations(File file, String serviceName) { SERVICE_CONFIG_MAP.put(serviceName, configs); } + private static void parseServiceTemplates(File file, String serviceName) { + File templateFolder = new File(file.getAbsolutePath(), TEMPLATE_FOLDER); + if (templateFolder.exists()) { + for (File templateFile : + Optional.ofNullable(templateFolder.listFiles()).orElse(new File[0])) { + String filename = templateFile.getName(); + String content = FileUtils.readFile2Str(templateFile); + Map map = SERVICE_TEMPLATE_MAP.computeIfAbsent(serviceName, k -> new HashMap<>()); + map.put(filename, content); + } + } + } + private static void parseDag(File file) { File dependencyFile = new File(file.getAbsolutePath(), DEPENDENCY_FILE_NAME); if (dependencyFile.exists()) { diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/metainfo.xml index 7ce431aa..7d5b3d4c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/metainfo.xml @@ -68,6 +68,13 @@ + + + + mysql hadoop diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/template/hive-service.sh b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/template/hive-service.sh new file mode 100644 index 00000000..49048a35 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hive/template/hive-service.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dir=$(dirname $0) +$dir/hive --service $1 > /dev/null 2>&1 & +echo $! > $2 \ No newline at end of file diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java index f60a5f41..8a9560b6 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return FlinkSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java index a769fd2c..a67091cc 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return FlinkSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java index 2883ebc9..1bad6405 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java @@ -45,6 +45,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params, getComponentName()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java index 4f33aff0..21c663d5 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java index f4ae58c0..a91f60ac 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java @@ -43,6 +43,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params, getComponentName()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java index b6f89c93..69c0bf33 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java @@ -45,6 +45,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params, getComponentName()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java index b92f64b6..9c8649e7 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java @@ -45,6 +45,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params, getComponentName()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java index 2f7bd857..46310a7a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java @@ -45,6 +45,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params, getComponentName()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java index dc8ba46f..149f16a3 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java @@ -45,6 +45,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HadoopSetup.configure(params, getComponentName()); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HBaseClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HBaseClientScript.java index c8a37d02..394eff7a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HBaseClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HBaseClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HBaseSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HMasterScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HMasterScript.java index 0954ca93..77b288da 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HMasterScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HMasterScript.java @@ -46,6 +46,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HBaseSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HRegionServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HRegionServerScript.java index c8e5e918..6c8b1dd5 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HRegionServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hbase/HRegionServerScript.java @@ -46,6 +46,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HBaseSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveClientScript.java index 2456ddf0..5a2cc62c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HiveSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java index c1eafa11..30253fca 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveMetastoreScript.java @@ -53,6 +53,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HiveSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveParams.java index 37447951..176f9b4f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveParams.java @@ -52,9 +52,6 @@ public class HiveParams extends BigtopParams { private String llapCliLog4j2Content; private String llapDaemonLog4j2Content; - private final String hiveShellContent = - "dir=$(dirname $0)\n$dir/hive --service $1 > /dev/null 2>&1 &\necho $! > $2"; - public HiveParams(ComponentCommandPayload componentCommandPayload) { super(componentCommandPayload); globalParamsMap.put("java_home", javaHome()); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveServer2Script.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveServer2Script.java index 20b65b4e..6f03437c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveServer2Script.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveServer2Script.java @@ -47,6 +47,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return HiveSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveSetup.java index 6ba97337..dba9993a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hive/HiveSetup.java @@ -48,14 +48,6 @@ public static ShellResult configure(Params params) { LinuxFileUtils.createDirectories(hiveParams.getHiveLogDir(), hiveUser, hiveGroup, PERMISSION_755, true); LinuxFileUtils.createDirectories(hiveParams.getHivePidDir(), hiveUser, hiveGroup, PERMISSION_755, true); - LinuxFileUtils.toFile( - ConfigType.CONTENT, - MessageFormat.format("{0}/hive-service.sh", hiveParams.serviceHome() + "/bin"), - hiveUser, - hiveGroup, - Constants.PERMISSION_755, - hiveParams.getHiveShellContent()); - LinuxFileUtils.toFileByTemplate( hiveParams.hiveLimits(), MessageFormat.format("{0}/hive.conf", HadoopParams.LIMITS_CONF_DIR), diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java index ff18e43e..93012ae3 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java @@ -46,6 +46,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return KafkaSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java index 0ab3d0ca..420451df 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrInstanceScript.java @@ -47,6 +47,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SolrSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java index 6b1df772..91dc876c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SparkSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java index 32beacec..dbeefd09 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkHistoryServerScript.java @@ -46,6 +46,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SparkSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java index df9e9f09..b65dd9d0 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/spark/SparkThriftServerScript.java @@ -46,6 +46,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SparkSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java index c9384164..640370ab 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -60,6 +60,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return TezSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java index 8a340f53..88e8690d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return ZookeeperSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java index 37f72a16..5ea7d39d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java @@ -46,6 +46,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return ZookeeperSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java index e9bdc46a..3c7985dc 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java @@ -24,6 +24,7 @@ import org.apache.bigtop.manager.grpc.pojo.PackageInfo; import org.apache.bigtop.manager.grpc.pojo.PackageSpecificInfo; import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.grpc.pojo.TemplateInfo; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; import org.apache.bigtop.manager.stack.core.exception.StackException; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; @@ -116,6 +117,11 @@ public List packages() { return packageInfoList; } + @Override + public List templates() { + return this.payload.getTemplates(); + } + @Override public String javaHome() { String root = LocalSettings.cluster().getRootDir(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java index 99538115..231a9dd1 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.grpc.pojo.PackageInfo; import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.grpc.pojo.TemplateInfo; import org.apache.bigtop.manager.stack.core.spi.PrioritySPI; import java.util.List; @@ -36,6 +37,8 @@ public interface Params extends PrioritySPI { List packages(); + List templates(); + String javaHome(); String stackHome(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java index 46a5fc17..913d66cd 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java @@ -22,6 +22,9 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.grpc.pojo.PackageInfo; import org.apache.bigtop.manager.grpc.pojo.RepoInfo; +import org.apache.bigtop.manager.grpc.pojo.TemplateInfo; +import org.apache.bigtop.manager.stack.core.enums.ConfigType; +import org.apache.bigtop.manager.stack.core.spi.param.BaseParams; import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.TarballUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; @@ -35,6 +38,8 @@ import java.util.List; import java.util.Properties; +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; + @Slf4j public abstract class AbstractScript implements Script { @@ -68,6 +73,26 @@ public ShellResult add(Params params, Properties properties) { return ShellResult.success(); } + @Override + public ShellResult configure(Params params) { + List templates = params.templates(); + for (TemplateInfo template : templates) { + String dir = params.serviceHome() + "/" + template.getDest(); + String filename = dir + "/" + template.getSrc(); + LinuxFileUtils.createDirectories(dir, params.user(), params.group(), PERMISSION_755, true); + LinuxFileUtils.toFile( + ConfigType.CONTENT, + filename, + params.user(), + params.group(), + Constants.PERMISSION_755, + template.getContent(), + ((BaseParams) params).getGlobalParamsMap()); + } + + return ShellResult.success(); + } + @Override public ShellResult init(Params params) { return ShellResult.success(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelClientScript.java index 85da982d..1c734af7 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SeaTunnelSetup.config(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelMasterScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelMasterScript.java index 85d4d39d..d3610f41 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelMasterScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelMasterScript.java @@ -48,6 +48,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SeaTunnelSetup.config(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelWorkerScript.java b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelWorkerScript.java index e3c8680d..beb75a20 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelWorkerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-extra/src/main/java/org/apache/bigtop/manager/stack/extra/v1_0_0/seatunnel/SeaTunnelWorkerScript.java @@ -48,6 +48,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return SeaTunnelSetup.config(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaServerScript.java index c93dd959..db96af1b 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaServerScript.java @@ -45,6 +45,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return GrafanaSetup.config(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLClientScript.java index a6ff2973..46b7fe5d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLClientScript.java @@ -42,6 +42,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return MySQLSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java index bff5e436..351613fa 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/mysql/MySQLServerScript.java @@ -47,6 +47,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return MySQLSetup.configure(params); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java index d98c84be..e2bf18e2 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java @@ -43,6 +43,8 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { + super.configure(params); + return PrometheusSetup.config(params); }