diff --git a/bigtop-manager-agent/pom.xml b/bigtop-manager-agent/pom.xml index 08fdcfcf..7923b0f9 100644 --- a/bigtop-manager-agent/pom.xml +++ b/bigtop-manager-agent/pom.xml @@ -63,6 +63,11 @@ bigtop-manager-stack-bigtop + + org.apache.bigtop + bigtop-manager-stack-infra + + org.springframework.boot spring-boot-starter-web diff --git a/bigtop-manager-bom/pom.xml b/bigtop-manager-bom/pom.xml index d0a4ad40..72f101ad 100644 --- a/bigtop-manager-bom/pom.xml +++ b/bigtop-manager-bom/pom.xml @@ -36,9 +36,10 @@ 2.3.32 3.12.0 2.0 - 2.13.0 + 2.16.1 32.1.1-jre 4.4 + 1.26.2 4.4.0 3.0.16 4.0.2 @@ -128,6 +129,12 @@ ${commons-collections4.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + + com.google.guava guava diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java index a3b7a3cc..bf7951c9 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.entity.pojo.CustomCommandInfo; -import org.apache.bigtop.manager.common.message.entity.pojo.OSSpecificInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageSpecificInfo; import org.apache.bigtop.manager.common.message.entity.pojo.ScriptInfo; import lombok.Data; @@ -56,5 +56,5 @@ public class CommandPayload extends BasePayload { private String componentName; - private List osSpecifics; + private List packageSpecifics; } diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageInfo.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageInfo.java new file mode 100644 index 00000000..dfccbc53 --- /dev/null +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageInfo.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.common.message.entity.pojo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class PackageInfo { + + private String name; + + private String checksum; +} diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/OSSpecificInfo.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageSpecificInfo.java similarity index 92% rename from bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/OSSpecificInfo.java rename to bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageSpecificInfo.java index 9c061e85..13be0345 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/OSSpecificInfo.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageSpecificInfo.java @@ -23,11 +23,11 @@ import java.util.List; @Data -public class OSSpecificInfo { +public class PackageSpecificInfo { private List os; private List arch; - private List packages; + private List packages; } diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/RepoInfo.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/RepoInfo.java index c48c8b1d..4eb48905 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/RepoInfo.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/RepoInfo.java @@ -31,6 +31,8 @@ public class RepoInfo { private String repoName; + private String repoType; + private String baseUrl; private String os; diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java index 4f502bfe..d0a922a4 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java @@ -69,6 +69,9 @@ public class RepoPO extends BasePO { @Column(name = "repo_name") private String repoName; + @Column(name = "repo_type") + private String repoType; + @ManyToOne @JoinColumn(name = "cluster_id", foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) private ClusterPO clusterPO; diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java index f49a254d..51a7369b 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java @@ -64,14 +64,14 @@ public class ServicePO extends BasePO { @Column(name = "display_name") private String displayName; - @Column(name = "service_desc") + @Column(name = "service_desc", length = 1024) private String serviceDesc; @Column(name = "service_version") private String serviceVersion; - @Column(name = "os_specifics") - private String osSpecifics; + @Column(name = "package_specifics", length = 1024) + private String packageSpecifics; @Column(name = "service_user") private String serviceUser; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java index 8987db00..2dc180e9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java @@ -87,18 +87,23 @@ public void beforeRun() { @Override public void run() { - beforeRun(); - boolean success = true; - LinkedBlockingQueue queue = new LinkedBlockingQueue<>(stages); - while (!queue.isEmpty()) { - Stage stage = queue.poll(); - Boolean stageSuccess = stage.run(); - - if (!stageSuccess) { - success = false; - break; + + try { + beforeRun(); + + LinkedBlockingQueue queue = new LinkedBlockingQueue<>(stages); + while (!queue.isEmpty()) { + Stage stage = queue.poll(); + Boolean stageSuccess = stage.run(); + + if (!stageSuccess) { + success = false; + break; + } } + } catch (Exception e) { + success = false; } if (success) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java index 76bf6f5e..a5bd909d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java @@ -96,7 +96,7 @@ protected StageContext createStageContext(String serviceName, String componentNa protected List getTodoListForCommand(Command command) { try { - List orderedList = dag.topologicalSort(); + List orderedList = dag.getAllNodesList().isEmpty() ? new ArrayList<>() : dag.topologicalSort(); List componentNames = getComponentNames(); List componentCommandNames = new ArrayList<>(componentNames.stream() .map(x -> x + "-" + command.name().toUpperCase()) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java index 9e2b5890..fb6029ad 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java @@ -78,7 +78,7 @@ protected TaskContext createTaskContext(String hostname) { Map properties = new HashMap<>(); properties.put("customCommands", componentDTO.getCustomCommands()); - properties.put("osSpecifics", serviceDTO.getOsSpecifics()); + properties.put("packageSpecifics", serviceDTO.getPackageSpecifics()); properties.put("commandScript", componentDTO.getCommandScript()); taskContext.setProperties(properties); return taskContext; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java index 4b72b62c..fd303739 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java @@ -79,24 +79,31 @@ public void beforeRun() { @Override public Boolean run() { - beforeRun(); - - List> futures = new ArrayList<>(); - for (Task task : tasks) { - futures.add(CompletableFuture.supplyAsync(task::run)); + boolean allTaskSuccess; + + try { + beforeRun(); + + List> futures = new ArrayList<>(); + for (Task task : tasks) { + futures.add(CompletableFuture.supplyAsync(task::run)); + } + + List taskResults = futures.stream() + .map((future) -> { + try { + return future.get(); + } catch (Exception e) { + return false; + } + }) + .toList(); + + allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue); + } catch (Exception e) { + allTaskSuccess = false; } - List taskResults = futures.stream() - .map((future) -> { - try { - return future.get(); - } catch (Exception e) { - return false; - } - }) - .toList(); - - boolean allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue); if (allTaskSuccess) { onSuccess(); } else { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java index 7ab444f5..763bf0a1 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java @@ -20,7 +20,8 @@ import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; import org.apache.bigtop.manager.common.message.entity.pojo.CustomCommandInfo; -import org.apache.bigtop.manager.common.message.entity.pojo.OSSpecificInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageSpecificInfo; import org.apache.bigtop.manager.common.message.entity.pojo.ScriptInfo; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.dao.repository.HostComponentRepository; @@ -28,7 +29,8 @@ import org.apache.bigtop.manager.grpc.generated.CommandType; import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.CustomCommandDTO; -import org.apache.bigtop.manager.server.model.dto.OSSpecificDTO; +import org.apache.bigtop.manager.server.model.dto.PackageDTO; +import org.apache.bigtop.manager.server.model.dto.PackageSpecificDTO; import org.apache.bigtop.manager.server.model.dto.ScriptDTO; import java.util.ArrayList; @@ -67,7 +69,8 @@ protected CommandRequest getCommandRequest() { commandPayload.setCustomCommands( convertCustomCommandInfo((List) properties.get("customCommands"))); - commandPayload.setOsSpecifics(convertOSSpecificInfo((List) properties.get("osSpecifics"))); + commandPayload.setPackageSpecifics( + convertPackageSpecificInfo((List) properties.get("packageSpecifics"))); commandPayload.setCommandScript(convertScriptInfo((ScriptDTO) properties.get("commandScript"))); CommandRequest.Builder builder = CommandRequest.newBuilder(); @@ -90,21 +93,28 @@ private ScriptInfo convertScriptInfo(ScriptDTO scriptDTO) { return scriptInfo; } - private List convertOSSpecificInfo(List osSpecificDTOs) { - if (osSpecificDTOs == null) { + private List convertPackageSpecificInfo(List packageSpecificDTOList) { + if (packageSpecificDTOList == null) { return new ArrayList<>(); } - List osSpecificInfos = new ArrayList<>(); - for (OSSpecificDTO osSpecificDTO : osSpecificDTOs) { - OSSpecificInfo osSpecificInfo = new OSSpecificInfo(); - osSpecificInfo.setOs(osSpecificDTO.getOs()); - osSpecificInfo.setArch(osSpecificDTO.getArch()); - osSpecificInfo.setPackages(osSpecificDTO.getPackages()); - osSpecificInfos.add(osSpecificInfo); + List packageSpecificInfos = new ArrayList<>(); + for (PackageSpecificDTO packageSpecificDTO : packageSpecificDTOList) { + PackageSpecificInfo packageSpecificInfo = new PackageSpecificInfo(); + packageSpecificInfo.setOs(packageSpecificDTO.getOs()); + packageSpecificInfo.setArch(packageSpecificDTO.getArch()); + List packageInfoList = new ArrayList<>(); + for (PackageDTO packageDTO : packageSpecificDTO.getPackages()) { + PackageInfo packageInfo = new PackageInfo(); + packageInfo.setName(packageDTO.getName()); + packageInfo.setChecksum(packageDTO.getChecksum()); + packageInfoList.add(packageInfo); + } + packageSpecificInfo.setPackages(packageInfoList); + packageSpecificInfos.add(packageSpecificInfo); } - return osSpecificInfos; + return packageSpecificInfos; } private List convertCustomCommandInfo(List customCommandDTOs) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java index c0916ee5..fc750b76 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java @@ -71,17 +71,23 @@ public void beforeRun() { @Override public Boolean run() { - beforeRun(); + boolean taskSuccess; - CommandRequest.Builder builder = CommandRequest.newBuilder(getCommandRequest()); - builder.setTaskId(getTaskPO().getId()); - commandRequest = builder.build(); + try { + beforeRun(); - CommandServiceGrpc.CommandServiceBlockingStub stub = GrpcClient.getBlockingStub( - taskContext.getHostname(), CommandServiceGrpc.CommandServiceBlockingStub.class); - CommandReply reply = stub.exec(commandRequest); + CommandRequest.Builder builder = CommandRequest.newBuilder(getCommandRequest()); + builder.setTaskId(getTaskPO().getId()); + commandRequest = builder.build(); - boolean taskSuccess = reply != null && reply.getCode() == MessageConstants.SUCCESS_CODE; + CommandServiceGrpc.CommandServiceBlockingStub stub = GrpcClient.getBlockingStub( + taskContext.getHostname(), CommandServiceGrpc.CommandServiceBlockingStub.class); + CommandReply reply = stub.exec(commandRequest); + + taskSuccess = reply != null && reply.getCode() == MessageConstants.SUCCESS_CODE; + } catch (Exception e) { + taskSuccess = false; + } if (taskSuccess) { onSuccess(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java index bee6a7fd..320f204c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java @@ -41,7 +41,7 @@ public interface ServiceConverter { ServiceConverter INSTANCE = Mappers.getMapper(ServiceConverter.class); - @Mapping(target = "osSpecifics", source = "osSpecifics", qualifiedByName = "obj2Json") + @Mapping(target = "packageSpecifics", source = "packageSpecifics", qualifiedByName = "obj2Json") @Mapping(target = "requiredServices", source = "requiredServices", qualifiedByName = "obj2Json") @Mapping(target = "clusterPO", expression = "java(clusterPO)") ServicePO fromDTO2PO(ServiceDTO serviceDTO, @Context ClusterPO clusterPO); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java new file mode 100644 index 00000000..8cabb51a --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.model.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class PackageDTO { + + private String name; + + private String checksum; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/OSSpecificDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageSpecificDTO.java similarity index 93% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/OSSpecificDTO.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageSpecificDTO.java index 4160f071..23c81a4c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/OSSpecificDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageSpecificDTO.java @@ -23,11 +23,11 @@ import java.util.List; @Data -public class OSSpecificDTO { +public class PackageSpecificDTO { private List os; private List arch; - private List packages; + private List packages; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java index aee938d1..66839563 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/RepoDTO.java @@ -31,6 +31,8 @@ public class RepoDTO { private String repoName; + private String repoType; + private String baseUrl; private String os; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java index b0c5a325..cb742eee 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/ServiceDTO.java @@ -35,7 +35,7 @@ public class ServiceDTO { private String serviceUser; - private List osSpecifics; + private List packageSpecifics; private List components; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java index a1cac071..9ad94ac3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/StackDTO.java @@ -29,6 +29,8 @@ public class StackDTO { private String stackVersion; + private String scope; + private String root; private String userGroup; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/RepoReq.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/RepoReq.java index c2d26ca5..324dc978 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/RepoReq.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/RepoReq.java @@ -31,6 +31,8 @@ public class RepoReq { private String repoName; + private String repoType; + private String baseUrl; private String os; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/RepoVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/RepoVO.java index ee7ca0f5..5968ea02 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/RepoVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/RepoVO.java @@ -27,6 +27,8 @@ public class RepoVO { private String repoName; + private String repoType; + private String baseUrl; private String os; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java new file mode 100644 index 00000000..ca9b2bea --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageModel.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.server.stack.model; + +import lombok.Data; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; + +@Data +@XmlAccessorType(XmlAccessType.FIELD) +public class PackageModel { + + private String name; + + private String checksum; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/OSSpecificModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageSpecificModel.java similarity index 95% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/OSSpecificModel.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageSpecificModel.java index b2dc4230..a600b763 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/OSSpecificModel.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/PackageSpecificModel.java @@ -29,7 +29,7 @@ @Data @XmlAccessorType(XmlAccessType.FIELD) -public class OSSpecificModel { +public class PackageSpecificModel { @XmlElementWrapper(name = "operating-systems") @XmlElements(@XmlElement(name = "os")) @@ -41,5 +41,5 @@ public class OSSpecificModel { @XmlElementWrapper(name = "packages") @XmlElements(@XmlElement(name = "package")) - private List packages; + private List packages; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/RepoModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/RepoModel.java index 3ea48876..d759e4da 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/RepoModel.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/RepoModel.java @@ -38,6 +38,12 @@ public class RepoModel { @XmlElement(name = "repo-name") private String repoName; + /** + * Type of repositories, values can be 'binary', 'tarball'. + */ + @XmlElement(name = "repo-type") + private String repoType; + @XmlElement(name = "base-url") private String baseUrl; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java index 944bb0ad..00a1092d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/ServiceModel.java @@ -42,9 +42,9 @@ public class ServiceModel { private String user; - @XmlElementWrapper(name = "os-specifics") - @XmlElements(@XmlElement(name = "os-specific")) - private List osSpecifics; + @XmlElementWrapper(name = "package-specifics") + @XmlElements(@XmlElement(name = "package-specific")) + private List packageSpecifics; @XmlElementWrapper(name = "components") @XmlElements(@XmlElement(name = "component")) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java index 45aeb40e..a8965862 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/stack/model/StackModel.java @@ -37,11 +37,16 @@ public class StackModel { @XmlElement(name = "stack-version") private String stackVersion; - private String root; - @XmlElement(name = "user-group") private String userGroup; + /** + * The scope of the stack, should be one of 'global' or 'cluster'. + */ + private String scope; + + private String root; + private String packages; @XmlElement(name = "repo-template") diff --git a/bigtop-manager-server/src/main/resources/ddl/DaMeng-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/DaMeng-DDL-CREATE.sql index f2eabdd9..6e24db99 100644 --- a/bigtop-manager-server/src/main/resources/ddl/DaMeng-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/DaMeng-DDL-CREATE.sql @@ -188,7 +188,7 @@ CREATE TABLE "bigtop_manager"."service" ( "update_by" BIGINT NULL, "update_time" DATETIME NULL, "display_name" VARCHAR(255) NULL, - "os_specifics" VARCHAR(255) NULL, + "package_specifics" VARCHAR(255) NULL, "required_service" VARCHAR(255) NULL, "service_desc" VARCHAR(255) NULL, "service_group" VARCHAR(255) NULL, diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql index 1573200e..ba9147fc 100644 --- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql @@ -168,6 +168,7 @@ CREATE TABLE `repo` `base_url` VARCHAR(64) DEFAULT NULL, `repo_id` VARCHAR(32) DEFAULT NULL, `repo_name` VARCHAR(64) DEFAULT NULL, + `repo_type` VARCHAR(64) DEFAULT NULL, `create_time` DATETIME DEFAULT NULL, `update_time` DATETIME DEFAULT NULL, `create_by` BIGINT, @@ -256,9 +257,9 @@ CREATE TABLE `service` `create_by` BIGINT, `create_time` DATETIME, `display_name` VARCHAR(255), - `os_specifics` VARCHAR(255), + `package_specifics` VARCHAR(1024), `required_services` VARCHAR(255), - `service_desc` VARCHAR(255), + `service_desc` VARCHAR(1024), `service_name` VARCHAR(255), `service_user` VARCHAR(255), `service_version` VARCHAR(255), diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml index ea143593..53a82c37 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/metainfo.xml @@ -23,6 +23,7 @@ bigtop 3.3.0 hadoop + cluster /opt curl,wget - - Bigtop-3.3.0 - Bigtop-3.3.0 for CentOS-7 x86_64 - centos7 - x86_64 - http://123.56.2.244/bigtop/3.3.0/centos7/x86_64/ - - - Bigtop-3.3.0 - Bigtop-3.3.0 for CentOS-7 aarch64 - centos7 - aarch64 - https://bigtop-snapshot.s3.amazonaws.com/centos-7/$basearch - Bigtop-3.3.0 Bigtop-3.3.0 for RockyLinux-8 x86_64 + binary rocky8 x86_64 http://123.56.2.244/bigtop/3.3.0/rocky8/x86_64/ diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml index 55987163..c62639a8 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml @@ -58,8 +58,8 @@ - - + + centos7 rocky8 @@ -68,10 +68,12 @@ x86_64 - flink_3_3_0 + + flink_3_3_0 + - - + + yarn diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml index 52dd1fb2..5878f17f 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml @@ -98,8 +98,8 @@ - - + + centos7 rocky8 @@ -108,12 +108,18 @@ x86_64 - hadoop_3_3_0 - hadoop_3_3_0-client - hadoop_3_3_0-libhdfs + + hadoop_3_3_0 + + + hadoop_3_3_0-client + + + hadoop_3_3_0-libhdfs + - - + + zookeeper diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml index 93febf4f..a31f99c3 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml @@ -54,8 +54,8 @@ - - + + centos7 rocky8 @@ -64,10 +64,12 @@ x86_64 - kafka_3_3_0 + + kafka_3_3_0 + - - + + zookeeper diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml index 95d6894b..4ece698e 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml @@ -58,8 +58,8 @@ - - + + centos7 rocky8 @@ -68,12 +68,18 @@ x86_64 - hadoop_3_3_0-yarn - hadoop_3_3_0-hdfs - hadoop_3_3_0-mapreduce + + hadoop_3_3_0-yarn + + + hadoop_3_3_0-hdfs + + + hadoop_3_3_0-mapreduce + - - + + yarn diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml index 15581b29..753f5cf7 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml @@ -48,8 +48,8 @@ - - + + centos7 rocky8 @@ -58,10 +58,13 @@ x86_64 - solr_3_3_0 + + solr_3_3_0 + - - + + + zookeeper diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml index 86c9ed84..59aacf0e 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml @@ -40,8 +40,8 @@ - - + + centos7 rocky8 @@ -50,10 +50,12 @@ x86_64 - tez_3_3_0 + + tez_3_3_0 + - - + + hdfs diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml index f66e0ae3..6ccd7477 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml @@ -87,8 +87,8 @@ - - + + centos7 rocky8 @@ -97,12 +97,18 @@ x86_64 - hadoop_3_3_0-yarn - hadoop_3_3_0-hdfs - hadoop_3_3_0-mapreduce + + hadoop_3_3_0-yarn + + + hadoop_3_3_0-hdfs + + + hadoop_3_3_0-mapreduce + - - + + zookeeper diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml index 620bf9cc..df686c0f 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml @@ -54,8 +54,8 @@ - - + + centos7 rocky8 @@ -64,9 +64,11 @@ x86_64 - zookeeper_3_3_0 + + zookeeper_3_3_0 + - - + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml new file mode 100644 index 00000000..74a38f3d --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/metainfo.xml @@ -0,0 +1,41 @@ + + + + + + infra + 1.0.0 + hadoop + global + /opt + curl,wget + + + + Infra-1.0.0 + Infra-1.0.0 for RockyLinux-8 x86_64 + tarball + rocky8 + x86_64 + http://123.56.2.244/infra/1.0.0/tarballs/ + + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/configuration/prometheus.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/configuration/prometheus.xml new file mode 100644 index 00000000..03d7e8a3 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/configuration/prometheus.xml @@ -0,0 +1,51 @@ + + + + + + content + This is the freemarker template for prometheus.yml file + ` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['localhost:9090'] +]]> + + + longtext + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml new file mode 100644 index 00000000..92b2fdd5 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/metainfo.xml @@ -0,0 +1,65 @@ + + + + + + prometheus + Prometheus + + Prometheus, a Cloud Native Computing Foundation project, is a systems and service monitoring system. It + collects metrics from configured targets at given intervals, evaluates rule expressions, displays the + results, and can trigger alerts when specified conditions are observed. + + 2.54.0 + prometheus + + + + prometheus_server + Prometheus Server + master + 1+ + + org.apache.bigtop.manager.stack.infra.v1_0_0.prometheus.PrometheusServerScript + java + 1200 + + + + + + + + centos7 + rocky8 + + + x86_64 + + + + prometheus-2.54.0.linux-amd64.tar.gz + SHA-256:465e1393a0cca9705598f6ffaf96ffa78d0347808ab21386b0c6aaec2cf7aa13 + + + + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/order.json b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/order.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/prometheus/order.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java new file mode 100644 index 00000000..8ffe6946 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.param; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.core.param.BaseParams; + +import java.text.MessageFormat; + +public abstract class BigtopParams extends BaseParams { + + protected BigtopParams(CommandPayload commandPayload) { + super(commandPayload); + } + + public String stackBinDir() { + String stackName = this.commandPayload.getStackName(); + String stackVersion = this.commandPayload.getStackVersion(); + String root = this.commandPayload.getRoot(); + return MessageFormat.format("{0}/{1}/{2}/usr/bin", root, stackName.toLowerCase(), stackVersion); + } + + public String stackLibDir() { + String stackName = this.commandPayload.getStackName(); + String stackVersion = this.commandPayload.getStackVersion(); + String root = this.commandPayload.getRoot(); + return MessageFormat.format("{0}/{1}/{2}/usr/lib", root, stackName.toLowerCase(), stackVersion); + } + + /** + * service home dir + */ + @Override + public String serviceHome() { + String service = this.commandPayload.getServiceName(); + return stackLibDir() + "/" + service.toLowerCase(); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java index cdda7670..fdfaccd5 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class FlinkClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return FlinkSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java index e4dc0636..34663933 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class FlinkHistoryServerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return FlinkSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java index 8ae12cdc..6c8b0b4c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.flink; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; @@ -31,7 +31,7 @@ @Getter @Slf4j -public class FlinkParams extends BaseParams { +public class FlinkParams extends BigtopParams { private String flinkLogDir; private String flinkPidDir; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java index 2085834c..b48090f8 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -35,11 +34,6 @@ @AutoService(Script.class) public class DataNodeScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return HdfsSetup.config(params, "datanode"); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java index 02262e7c..455d9400 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class HdfsClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return HdfsSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java index aa45a5d2..df7122ac 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.commons.lang3.StringUtils; @@ -35,7 +35,7 @@ @Getter @Slf4j -public class HdfsParams extends BaseParams { +public class HdfsParams extends BigtopParams { private String hadoopLogDir = "/var/log/hadoop"; private String hadoopPidDir = "/var/run/hadoop"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java index 5f30f29f..2172414a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -35,11 +34,6 @@ @AutoService(Script.class) public class NameNodeScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return HdfsSetup.config(params, "namenode"); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java index 67eba56f..c92a2831 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -35,11 +34,6 @@ @AutoService(Script.class) public class SNameNodeScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return HdfsSetup.config(params, "secondarynamenode"); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java index 7cc29c48..308b102b 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -36,11 +35,6 @@ @AutoService(Script.class) public class KafkaBrokerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return KafkaSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java index c3c6411b..faa6ea1d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; @@ -28,7 +28,7 @@ import java.util.Map; @Getter -public class KafkaParams extends BaseParams { +public class KafkaParams extends BigtopParams { private String kafkaLogDir = "/var/log/kafka"; private String kafkaPidDir = "/var/run/kafka"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java index 58e5b39c..b07d4b9f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -33,11 +32,6 @@ @AutoService(Script.class) public class HistoryServerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return MapredSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java index 48429f5c..1aea9577 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class MapredClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return MapredSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java index f690b09f..f4491f6c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; @@ -31,7 +31,7 @@ @Getter @Slf4j -public class Mapreduce2Params extends BaseParams { +public class Mapreduce2Params extends BigtopParams { private String mapredEnvContent; private String mapredLogDir; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java index 23320fe2..e7a2b6ac 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class SolrClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return SolrSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java index 2f5a9f1b..4e68b79f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.solr; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; @@ -32,7 +32,7 @@ @Getter @Slf4j -public class SolrParams extends BaseParams { +public class SolrParams extends BigtopParams { private String solrPort = "8983"; private String solrLogDir = "/var/log/solr"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java index fdff7da4..dd93fa79 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -36,11 +35,6 @@ @AutoService(Script.class) public class SolrServerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return SolrSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java index 5db7b467..832ada59 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class TezClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return TezSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java index f74724d5..261fd0e2 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.commons.lang3.StringUtils; @@ -33,7 +33,7 @@ @Getter @Slf4j -public class TezParams extends BaseParams { +public class TezParams extends BigtopParams { private String headDumpOpts; private String tezEnvContent; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java index 8f89d711..ac90d6b9 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -35,11 +34,6 @@ @AutoService(Script.class) public class NodeManagerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return YarnSetup.config(params, "nodemanager"); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java index 0db3b118..416b69fe 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -35,11 +34,6 @@ @AutoService(Script.class) public class ResourceManagerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return YarnSetup.config(params, "resourcemanager"); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java index 10e3d006..cbd29d0e 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class YarnClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return YarnSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java index e92dafab..24fbb1db 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; @@ -32,7 +32,7 @@ @Getter @Slf4j -public class YarnParams extends BaseParams { +public class YarnParams extends BigtopParams { private String yarnLogDir = "/var/log/hadoop-yarn"; private String yarnPidDir = "/var/run/hadoop-yarn"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java index 067ef09d..87faa209 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java @@ -22,7 +22,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -31,11 +30,6 @@ @AutoService(Script.class) public class ZookeeperClientScript extends AbstractClientScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return ZookeeperSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java index f74faba7..05678f73 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.zookeeper; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.param.BaseParams; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; @@ -28,7 +28,7 @@ import java.util.Map; @Getter -public class ZookeeperParams extends BaseParams { +public class ZookeeperParams extends BigtopParams { private String zookeeperLogDir = "/var/log/zookeeper"; private String zookeeperPidDir = "/var/run/zookeeper"; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java index 4483094c..0a7ed502 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java @@ -23,7 +23,6 @@ import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; -import org.apache.bigtop.manager.stack.core.utils.PackageUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; import com.google.auto.service.AutoService; @@ -36,11 +35,6 @@ @AutoService(Script.class) public class ZookeeperServerScript extends AbstractServerScript { - @Override - public ShellResult install(Params params) { - return PackageUtils.install(params.getPackageList()); - } - @Override public ShellResult configure(Params params) { return ZookeeperSetup.config(params); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/pom.xml b/bigtop-manager-stack/bigtop-manager-stack-core/pom.xml index 0423dce5..d9591a6b 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/pom.xml +++ b/bigtop-manager-stack/bigtop-manager-stack-core/pom.xml @@ -49,5 +49,10 @@ commons-io commons-io + + + org.apache.commons + commons-compress + diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java index 49bd5ce9..4a4111bd 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java @@ -20,10 +20,13 @@ import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; -import org.apache.bigtop.manager.common.message.entity.pojo.OSSpecificInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageSpecificInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; import org.apache.bigtop.manager.common.utils.NetUtils; import org.apache.bigtop.manager.common.utils.os.OSDetection; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.exception.StackException; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.commons.lang3.StringUtils; @@ -33,6 +36,7 @@ import java.lang.reflect.Method; import java.text.MessageFormat; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -72,72 +76,66 @@ protected BaseParams(CommandPayload commandPayload) { globalParamsMap.remove("content"); } - /** - * get the package list according to the os and arch - */ - public List getPackageList() { - List osSpecifics = this.commandPayload.getOsSpecifics(); - if (osSpecifics == null) { - return List.of(); - } - - String os = OSDetection.getOS(); - String arch = OSDetection.getArch(); - for (OSSpecificInfo osSpecific : osSpecifics) { - List pkgOS = osSpecific.getOs(); - List pkgArch = osSpecific.getArch(); - if (pkgOS.contains(os) && pkgArch.contains(arch)) { - return osSpecific.getPackages(); - } - } - - return List.of(); - } - public String hostname() { return NetUtils.getHostname(); } - public String stackBinDir() { - String stackName = this.commandPayload.getStackName(); - String stackVersion = this.commandPayload.getStackVersion(); - String root = this.commandPayload.getRoot(); - return MessageFormat.format("{0}/{1}/{2}/usr/bin", root, stackName.toLowerCase(), stackVersion); - } - - public String stackLibDir() { - String stackName = this.commandPayload.getStackName(); - String stackVersion = this.commandPayload.getStackVersion(); - String root = this.commandPayload.getRoot(); - return MessageFormat.format("{0}/{1}/{2}/usr/lib", root, stackName.toLowerCase(), stackVersion); - } - - /** - * service home dir - */ - public String serviceHome() { - String service = this.commandPayload.getServiceName(); - return stackLibDir() + "/" + service.toLowerCase(); - } - /** * service conf dir */ + @Override public String confDir() { return "/etc/" + this.commandPayload.getServiceName().toLowerCase() + "/conf"; } + @Override public String user() { return StringUtils.isNotBlank(this.commandPayload.getServiceUser()) ? this.commandPayload.getServiceUser() : ROOT_USER; } + @Override public String group() { return LocalSettings.cluster().getUserGroup(); } + @Override + public RepoInfo repo() { + return LocalSettings.repos().stream() + .filter(r -> OSDetection.getOS().equals(r.getOs()) + && OSDetection.getArch().equals(r.getArch())) + .findFirst() + .orElseThrow(() -> new StackException( + "Cannot find repo for os: [{}] and arch: [{}]", OSDetection.getOS(), OSDetection.getArch())); + } + + @Override + public List packages() { + RepoInfo repo = this.repo(); + List packageInfoList = new ArrayList<>(); + for (PackageSpecificInfo packageSpecificInfo : this.commandPayload.getPackageSpecifics()) { + if (!packageSpecificInfo.getOs().contains(repo.getOs()) + || !packageSpecificInfo.getArch().contains(repo.getArch())) { + continue; + } + + packageInfoList.addAll(packageSpecificInfo.getPackages()); + } + + return packageInfoList; + } + + @Override public String serviceName() { return this.commandPayload.getServiceName(); } + + @Override + public String stackHome() { + String stackName = this.commandPayload.getStackName(); + String stackVersion = this.commandPayload.getStackVersion(); + String root = this.commandPayload.getRoot(); + return MessageFormat.format("{0}/{1}/{2}", root, stackName.toLowerCase(), stackVersion); + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java index fdaf10fd..10283295 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java @@ -18,19 +18,26 @@ */ package org.apache.bigtop.manager.stack.core.param; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; + import java.util.List; public interface Params { - List getPackageList(); - - String serviceHome(); - String confDir(); String user(); String group(); + RepoInfo repo(); + + List packages(); + String serviceName(); + + String serviceHome(); + + String stackHome(); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/InstallHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/InstallHook.java index c10279de..1015a106 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/InstallHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/InstallHook.java @@ -75,7 +75,8 @@ private void addUserAndGroup(Params params) { private void writeRepo(Params params) { RepoInfo repo = LocalSettings.repos().stream() .filter(r -> OSDetection.getOS().equals(r.getOs()) - && OSDetection.getArch().equals(r.getArch())) + && OSDetection.getArch().equals(r.getArch()) + && !r.getRepoType().equals("tarball")) .findFirst() .orElse(null); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java index 535d9872..3342a3b1 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java @@ -18,16 +18,64 @@ */ package org.apache.bigtop.manager.stack.core.spi.script; +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.utils.PackageUtils; +import org.apache.bigtop.manager.stack.core.utils.TarballUtils; import org.apache.commons.lang3.StringUtils; import lombok.extern.slf4j.Slf4j; +import java.util.List; +import java.util.Properties; + @Slf4j public abstract class AbstractScript implements Script { + public static final String REPO_TYPE_BINARY = "binary"; + public static final String REPO_TYPE_TARBALL = "tarball"; + + public static final String PROPERTY_KEY_SKIP_LEVELS = "skipLevels"; + + @Override + public ShellResult install(Params params) { + return this.install(params, new Properties()); + } + + public ShellResult install(Params params, Properties properties) { + if (params.repo().getRepoType().equals(REPO_TYPE_BINARY)) { + return installBinary(params); + } else if (params.repo().getRepoType().equals(REPO_TYPE_TARBALL)) { + return installTarball(params, properties); + } else { + log.error("Unsupported repo type: {}", params.repo().getRepoType()); + return ShellResult.fail(); + } + } + + private ShellResult installBinary(Params params) { + List packages = params.packages(); + List packageNames = packages.stream().map(PackageInfo::getName).toList(); + return PackageUtils.install(packageNames); + } + + private ShellResult installTarball(Params params, Properties properties) { + RepoInfo repo = params.repo(); + List packages = params.packages(); + String stackHome = params.stackHome(); + String serviceHome = params.serviceHome(); + + for (PackageInfo packageInfo : packages) { + Integer skipLevels = Integer.parseInt(properties.getProperty(PROPERTY_KEY_SKIP_LEVELS, "0")); + TarballUtils.installPackage(repo.getBaseUrl(), stackHome, serviceHome, packageInfo, skipLevels); + } + + return ShellResult.success(); + } + public ShellResult restart(Params params) { ShellResult shellResult = stop(params); if (shellResult.getExitCode() != 0) { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/ChecksumValidator.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/ChecksumValidator.java new file mode 100644 index 00000000..81b407f7 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/ChecksumValidator.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.core.tarball; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.FileInputStream; +import java.security.MessageDigest; + +@Slf4j +public class ChecksumValidator { + + public static boolean validateChecksum(String algorithm, String value, File file) { + try { + MessageDigest digest = MessageDigest.getInstance(algorithm.toUpperCase()); + FileInputStream fis = new FileInputStream(file); + byte[] byteArray = new byte[1024]; + int bytesCount; + + while ((bytesCount = fis.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount); + } + fis.close(); + + byte[] bytes = digest.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + + String calculatedChecksum = sb.toString(); + return calculatedChecksum.equalsIgnoreCase(value); + } catch (Exception e) { + log.error("Error while validating checksum", e); + return false; + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java new file mode 100644 index 00000000..ace5bae8 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.core.tarball; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; + +@Slf4j +public class TarballDownloader { + + @SuppressWarnings("ResultOfMethodCallIgnored") + public static Boolean downloadFile(String fileUrl, String saveDir) { + HttpURLConnection httpConn = null; + try { + URL url = new URL(fileUrl); + httpConn = (HttpURLConnection) url.openConnection(); + int responseCode = httpConn.getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_OK) { + String fileName = ""; + String disposition = httpConn.getHeaderField("Content-Disposition"); + + if (disposition != null) { + int index = disposition.indexOf("filename="); + if (index > 0) { + fileName = disposition.substring(index + 10, disposition.length() - 1); + } + } else { + fileName = fileUrl.substring(fileUrl.lastIndexOf("/") + 1); + } + + InputStream inputStream = httpConn.getInputStream(); + String saveFilePath = saveDir + File.separator + fileName; + + if (!new File(saveDir).exists()) { + new File(saveDir).mkdirs(); + } + + FileOutputStream outputStream = new FileOutputStream(saveFilePath); + + int bytesRead = -1; + byte[] buffer = new byte[4096]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + inputStream.close(); + + log.info("File downloaded: {}", saveFilePath); + return true; + } else { + log.info("No file to download. Server replied HTTP code: {}", responseCode); + return false; + } + } catch (Exception e) { + log.error("Error downloading file: {}", e.getMessage()); + return false; + } finally { + if (httpConn != null) { + httpConn.disconnect(); + } + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java new file mode 100644 index 00000000..e087028e --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.core.tarball; + +import org.apache.bigtop.manager.stack.core.exception.StackException; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.function.Function; + +@Slf4j +public class TarballExtractor { + + public static void extractTarball(String source, String dest) { + extractTarball(source, dest, 0); + } + + public static void extractTarball(String source, String dest, Integer skipLevels) { + File tarball = new File(source); + File destDir = new File(dest); + + if (isTarGz(source)) { + extractTarGz(tarball, tis -> extract(tis, destDir, skipLevels)); + } else if (isTar(source)) { + extractTar(tarball, tis -> extract(tis, destDir, skipLevels)); + } else { + log.info("Unsupported file type: {}", source); + } + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static Boolean extract(TarArchiveInputStream tis, File destDir, Integer skipLevels) { + try { + TarArchiveEntry entry; + while ((entry = tis.getNextEntry()) != null) { + String entryName = entry.getName(); + String[] parts = entryName.split("/"); + if (parts.length <= skipLevels) { + continue; // Skip this entry + } + StringBuilder newName = new StringBuilder(); + for (int i = skipLevels; i < parts.length; i++) { + newName.append(parts[i]); + if (i < parts.length - 1) { + newName.append(File.separator); + } + } + File outputFile = new File(destDir, newName.toString()); + + if (entry.isDirectory()) { + if (!outputFile.exists()) { + outputFile.mkdirs(); + } + } else { + File parent = outputFile.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } + + try (FileOutputStream fos = new FileOutputStream(outputFile)) { + byte[] buffer = new byte[1024]; + int len; + while ((len = tis.read(buffer)) != -1) { + fos.write(buffer, 0, len); + } + } + } + } + } catch (Exception e) { + log.info("Error extracting tarball", e); + throw new StackException(e); + } + + return true; + } + + private static void extractTar(File tarball, Function func) { + try (InputStream fis = Files.newInputStream(tarball.toPath()); + TarArchiveInputStream tis = new TarArchiveInputStream(fis)) { + func.apply(tis); + } catch (Exception e) { + log.error("Error extracting tarball", e); + throw new StackException(e); + } + } + + private static void extractTarGz(File tarball, Function func) { + try (InputStream fis = Files.newInputStream(tarball.toPath()); + GzipCompressorInputStream gis = new GzipCompressorInputStream(fis); + TarArchiveInputStream tis = new TarArchiveInputStream(gis)) { + func.apply(tis); + } catch (Exception e) { + log.error("Error extracting tarball", e); + throw new StackException(e); + } + } + + private static boolean isTarGz(String filePath) { + return filePath.endsWith(".tar.gz"); + } + + private static boolean isTar(String filePath) { + return filePath.endsWith(".tar"); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java new file mode 100644 index 00000000..cb6bf087 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.core.utils; + +import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.tarball.ChecksumValidator; +import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader; +import org.apache.bigtop.manager.stack.core.tarball.TarballExtractor; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; + +@Slf4j +public class TarballUtils { + + @SuppressWarnings("ResultOfMethodCallIgnored") + public static void installPackage( + String repoUrl, String stackHome, String serviceHome, PackageInfo packageInfo, Integer skipLevels) { + String remoteUrl = repoUrl + File.separator + packageInfo.getName(); + File localFile = new File(stackHome + File.separator + packageInfo.getName()); + String algorithm = packageInfo.getChecksum().split(":")[0]; + String checksum = packageInfo.getChecksum().split(":")[1]; + + if (localFile.exists()) { + log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath()); + } else { + log.info("Downloading [{}] to [{}]", remoteUrl, stackHome); + downloadPackage(remoteUrl, stackHome); + } + + boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile); + if (!validateChecksum) { + log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath()); + localFile.delete(); + downloadPackage(remoteUrl, stackHome); + } + + validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile); + if (!validateChecksum) { + log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath()); + throw new StackException("Invalid checksum for " + localFile.getAbsolutePath()); + } + + log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath()); + log.info("Extracting [{}] to [{}]", localFile.getAbsolutePath(), serviceHome); + TarballExtractor.extractTarball(localFile.getAbsolutePath(), serviceHome, skipLevels); + log.info("File [{}] successfully extracted to [{}]", localFile.getAbsolutePath(), serviceHome); + } + + private static void downloadPackage(String remoteUrl, String saveDir) { + int i = 1; + while (true) { + Boolean downloaded = TarballDownloader.downloadFile(remoteUrl, saveDir); + if (downloaded) { + break; + } else { + if (i == 3) { + log.error("Failed to download [{}], exiting...", remoteUrl); + throw new StackException("Failed to download " + remoteUrl); + } else { + log.error("Failed to download [{}], retrying...: {}", remoteUrl, i); + } + } + + i++; + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java index fcf70b22..a143afae 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java @@ -224,8 +224,8 @@ public static void createDirectories( log.error("dirPath must not be null"); return; } - Path path = Paths.get(dirPath); + Path path = Paths.get(dirPath); if (Files.isSymbolicLink(path)) { log.error("Directory is symbolic link: [{}]", dirPath); return; @@ -233,7 +233,15 @@ public static void createDirectories( try { log.info("Creating directory: [{}]", path); - Files.createDirectories(path); + if (Files.exists(path)) { + if (Files.isDirectory(path)) { + log.info("Directory already exists: [{}], skip creating", path); + } else { + throw new IOException("Path exists and is not a directory: " + path); + } + } else { + Files.createDirectories(path); + } } catch (IOException e) { log.error("Error when create directory", e); } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/pom.xml b/bigtop-manager-stack/bigtop-manager-stack-infra/pom.xml new file mode 100644 index 00000000..a7e0c800 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + org.apache.bigtop + bigtop-manager-stack + ${revision} + + + bigtop-manager-stack-infra + ${project.artifactId} + Bigtop Manager Stack Infra + + + + org.apache.bigtop + bigtop-manager-stack-core + + + + diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java new file mode 100644 index 00000000..f78f8058 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.infra.param; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.core.param.BaseParams; + +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; + +@Slf4j +public abstract class InfraParams extends BaseParams { + + protected InfraParams(CommandPayload commandPayload) { + super(commandPayload); + } + + /** + * service home dir + */ + @Override + public String serviceHome() { + String service = this.commandPayload.getServiceName(); + return MessageFormat.format("{0}/{1}", stackHome(), service.toLowerCase()); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java new file mode 100644 index 00000000..7c96773c --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.infra.v1_0_0.prometheus; + +import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.infra.param.InfraParams; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; + +@Getter +@Slf4j +public class PrometheusParams extends InfraParams { + + public PrometheusParams(CommandPayload commandPayload) { + super(commandPayload); + } + + public String dataDir() { + return MessageFormat.format("{0}/data", serviceHome()); + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java new file mode 100644 index 00000000..581ddb67 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.infra.v1_0_0.prometheus; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; +import org.apache.bigtop.manager.stack.core.spi.script.Script; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; + +import com.google.auto.service.AutoService; + +import java.text.MessageFormat; +import java.util.Properties; + +@AutoService(Script.class) +public class PrometheusServerScript extends AbstractServerScript { + + @Override + public ShellResult install(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.install(params, properties); + } + + @Override + public ShellResult configure(Params params) { + return PrometheusSetup.config(params); + } + + @Override + public ShellResult start(Params params) { + configure(params); + PrometheusParams prometheusParams = (PrometheusParams) params; + String cmd = MessageFormat.format( + "nohup {0}/prometheus --config.file={0}/prometheus.yml --storage.tsdb.path={0}/data > {0}/nohup.out 2>&1 &", + prometheusParams.serviceHome()); + try { + return LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + + @Override + public ShellResult stop(Params params) { + PrometheusParams prometheusParams = (PrometheusParams) params; + String cmd = "pkill -f prometheus"; + try { + return LinuxOSUtils.sudoExecCmd(cmd, prometheusParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + } + + @Override + public ShellResult status(Params params) { + String cmd = "pgrep -f prometheus"; + try { + ShellResult result = LinuxOSUtils.execCmd(cmd); + if (result.getExitCode() == 0) { + return ShellResult.success(); + } else { + return new ShellResult(-1, "", "Prometheus is not running"); + } + } catch (Exception e) { + throw new StackException(e); + } + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java new file mode 100644 index 00000000..0a1eb357 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.infra.v1_0_0.prometheus; + +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class PrometheusSetup { + + public static ShellResult config(Params params) { + PrometheusParams prometheusParams = (PrometheusParams) params; + String user = prometheusParams.user(); + String group = prometheusParams.group(); + + LinuxFileUtils.createDirectories(prometheusParams.serviceHome(), user, group, PERMISSION_755, true); + LinuxFileUtils.createDirectories(prometheusParams.dataDir(), user, group, PERMISSION_755, true); + + return ShellResult.success("Prometheus Configure success!"); + } +} diff --git a/bigtop-manager-stack/pom.xml b/bigtop-manager-stack/pom.xml index 265065f6..9334c1b2 100644 --- a/bigtop-manager-stack/pom.xml +++ b/bigtop-manager-stack/pom.xml @@ -35,6 +35,7 @@ bigtop-manager-stack-core bigtop-manager-stack-bigtop + bigtop-manager-stack-infra diff --git a/bigtop-manager-ui/src/api/repo/types.ts b/bigtop-manager-ui/src/api/repo/types.ts index 46e1ce6e..ade42756 100644 --- a/bigtop-manager-ui/src/api/repo/types.ts +++ b/bigtop-manager-ui/src/api/repo/types.ts @@ -20,6 +20,7 @@ export interface RepoVO { repoId: string repoName: string + repoType: string baseUrl: string os: string arch: string diff --git a/pom.xml b/pom.xml index dfb4d99c..66f14b04 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,12 @@ ${project.version} + + org.apache.bigtop + bigtop-manager-stack-infra + ${project.version} + + org.apache.bigtop bigtop-manager-ui