BIGTOP-4194: Add infra stack with the simplest Prometheus support (#51)

bigtop-manager-agent/pom.xml

@@ -63,6 +63,11 @@
             <artifactId>bigtop-manager-stack-bigtop</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.bigtop</groupId>
+            <artifactId>bigtop-manager-stack-infra</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>

bigtop-manager-bom/pom.xml

@@ -36,9 +36,10 @@
         <freemarker.version>2.3.32</freemarker.version>
         <common-lang3.version>3.12.0</common-lang3.version>
         <snakeyaml.version>2.0</snakeyaml.version>
-        <commons-io.version>2.13.0</commons-io.version>
+        <commons-io.version>2.16.1</commons-io.version>
         <guava.version>32.1.1-jre</guava.version>
         <commons-collections4.version>4.4</commons-collections4.version>
+        <commons-compress.version>1.26.2</commons-compress.version>
         <jwt.version>4.4.0</jwt.version>
         <janino.version>3.0.16</janino.version>
         <eclipse-link.version>4.0.2</eclipse-link.version>
@@ -128,6 +129,12 @@
                 <version>${commons-collections4.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>${commons-compress.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>com.google.guava</groupId>
                 <artifactId>guava</artifactId>

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java

@@ -20,7 +20,7 @@
 
 import org.apache.bigtop.manager.common.enums.Command;
 import org.apache.bigtop.manager.common.message.entity.pojo.CustomCommandInfo;
-import org.apache.bigtop.manager.common.message.entity.pojo.OSSpecificInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageSpecificInfo;
 import org.apache.bigtop.manager.common.message.entity.pojo.ScriptInfo;
 
 import lombok.Data;
@@ -56,5 +56,5 @@ public class CommandPayload extends BasePayload {
 
     private String componentName;
 
-    private List<OSSpecificInfo> osSpecifics;
+    private List<PackageSpecificInfo> packageSpecifics;
 }

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageInfo.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.common.message.entity.pojo;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class PackageInfo {
+
+    private String name;
+
+    private String checksum;
+}

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/OSSpecificInfo.java → bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/PackageSpecificInfo.java

@@ -23,11 +23,11 @@
 import java.util.List;
 
 @Data
-public class OSSpecificInfo {
+public class PackageSpecificInfo {
 
     private List<String> os;
 
     private List<String> arch;
 
-    private List<String> packages;
+    private List<PackageInfo> packages;
 }

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/pojo/RepoInfo.java

@@ -31,6 +31,8 @@ public class RepoInfo {
 
     private String repoName;
 
+    private String repoType;
+
     private String baseUrl;
 
     private String os;

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java

@@ -69,6 +69,9 @@ public class RepoPO extends BasePO {
     @Column(name = "repo_name")
     private String repoName;
 
+    @Column(name = "repo_type")
+    private String repoType;
+
     @ManyToOne
     @JoinColumn(name = "cluster_id", foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT))
     private ClusterPO clusterPO;

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java

@@ -64,14 +64,14 @@ public class ServicePO extends BasePO {
     @Column(name = "display_name")
     private String displayName;
 
-    @Column(name = "service_desc")
+    @Column(name = "service_desc", length = 1024)
     private String serviceDesc;
 
     @Column(name = "service_version")
     private String serviceVersion;
 
-    @Column(name = "os_specifics")
-    private String osSpecifics;
+    @Column(name = "package_specifics", length = 1024)
+    private String packageSpecifics;
 
     @Column(name = "service_user")
     private String serviceUser;

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java

@@ -87,18 +87,23 @@ public void beforeRun() {
 
     @Override
     public void run() {
-        beforeRun();
-
         boolean success = true;
-        LinkedBlockingQueue<Stage> queue = new LinkedBlockingQueue<>(stages);
-        while (!queue.isEmpty()) {
-            Stage stage = queue.poll();
-            Boolean stageSuccess = stage.run();
-
-            if (!stageSuccess) {
-                success = false;
-                break;
+
+        try {
+            beforeRun();
+
+            LinkedBlockingQueue<Stage> queue = new LinkedBlockingQueue<>(stages);
+            while (!queue.isEmpty()) {
+                Stage stage = queue.poll();
+                Boolean stageSuccess = stage.run();
+
+                if (!stageSuccess) {
+                    success = false;
+                    break;
+                }
             }
+        } catch (Exception e) {
+            success = false;
         }
 
         if (success) {

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractServiceJob.java

@@ -96,7 +96,7 @@ protected StageContext createStageContext(String serviceName, String componentNa
 
     protected List<String> getTodoListForCommand(Command command) {
         try {
-            List<String> orderedList = dag.topologicalSort();
+            List<String> orderedList = dag.getAllNodesList().isEmpty() ? new ArrayList<>() : dag.topologicalSort();
             List<String> componentNames = getComponentNames();
             List<String> componentCommandNames = new ArrayList<>(componentNames.stream()
                     .map(x -> x + "-" + command.name().toUpperCase())

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java

@@ -78,7 +78,7 @@ protected TaskContext createTaskContext(String hostname) {
 
         Map<String, Object> properties = new HashMap<>();
         properties.put("customCommands", componentDTO.getCustomCommands());
-        properties.put("osSpecifics", serviceDTO.getOsSpecifics());
+        properties.put("packageSpecifics", serviceDTO.getPackageSpecifics());
         properties.put("commandScript", componentDTO.getCommandScript());
         taskContext.setProperties(properties);
         return taskContext;

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java

@@ -79,24 +79,31 @@ public void beforeRun() {
 
     @Override
     public Boolean run() {
-        beforeRun();
-
-        List<CompletableFuture<Boolean>> futures = new ArrayList<>();
-        for (Task task : tasks) {
-            futures.add(CompletableFuture.supplyAsync(task::run));
+        boolean allTaskSuccess;
+
+        try {
+            beforeRun();
+
+            List<CompletableFuture<Boolean>> futures = new ArrayList<>();
+            for (Task task : tasks) {
+                futures.add(CompletableFuture.supplyAsync(task::run));
+            }
+
+            List<Boolean> taskResults = futures.stream()
+                    .map((future) -> {
+                        try {
+                            return future.get();
+                        } catch (Exception e) {
+                            return false;
+                        }
+                    })
+                    .toList();
+
+            allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue);
+        } catch (Exception e) {
+            allTaskSuccess = false;
         }
 
-        List<Boolean> taskResults = futures.stream()
-                .map((future) -> {
-                    try {
-                        return future.get();
-                    } catch (Exception e) {
-                        return false;
-                    }
-                })
-                .toList();
-
-        boolean allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue);
         if (allTaskSuccess) {
             onSuccess();
         } else {

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java

@@ -20,15 +20,17 @@
 
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
 import org.apache.bigtop.manager.common.message.entity.pojo.CustomCommandInfo;
-import org.apache.bigtop.manager.common.message.entity.pojo.OSSpecificInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageSpecificInfo;
 import org.apache.bigtop.manager.common.message.entity.pojo.ScriptInfo;
 import org.apache.bigtop.manager.common.utils.JsonUtils;
 import org.apache.bigtop.manager.dao.repository.HostComponentRepository;
 import org.apache.bigtop.manager.grpc.generated.CommandRequest;
 import org.apache.bigtop.manager.grpc.generated.CommandType;
 import org.apache.bigtop.manager.server.holder.SpringContextHolder;
 import org.apache.bigtop.manager.server.model.dto.CustomCommandDTO;
-import org.apache.bigtop.manager.server.model.dto.OSSpecificDTO;
+import org.apache.bigtop.manager.server.model.dto.PackageDTO;
+import org.apache.bigtop.manager.server.model.dto.PackageSpecificDTO;
 import org.apache.bigtop.manager.server.model.dto.ScriptDTO;
 
 import java.util.ArrayList;
@@ -67,7 +69,8 @@ protected CommandRequest getCommandRequest() {
 
         commandPayload.setCustomCommands(
                 convertCustomCommandInfo((List<CustomCommandDTO>) properties.get("customCommands")));
-        commandPayload.setOsSpecifics(convertOSSpecificInfo((List<OSSpecificDTO>) properties.get("osSpecifics")));
+        commandPayload.setPackageSpecifics(
+                convertPackageSpecificInfo((List<PackageSpecificDTO>) properties.get("packageSpecifics")));
         commandPayload.setCommandScript(convertScriptInfo((ScriptDTO) properties.get("commandScript")));
 
         CommandRequest.Builder builder = CommandRequest.newBuilder();
@@ -90,21 +93,28 @@ private ScriptInfo convertScriptInfo(ScriptDTO scriptDTO) {
         return scriptInfo;
     }
 
-    private List<OSSpecificInfo> convertOSSpecificInfo(List<OSSpecificDTO> osSpecificDTOs) {
-        if (osSpecificDTOs == null) {
+    private List<PackageSpecificInfo> convertPackageSpecificInfo(List<PackageSpecificDTO> packageSpecificDTOList) {
+        if (packageSpecificDTOList == null) {
             return new ArrayList<>();
         }
 
-        List<OSSpecificInfo> osSpecificInfos = new ArrayList<>();
-        for (OSSpecificDTO osSpecificDTO : osSpecificDTOs) {
-            OSSpecificInfo osSpecificInfo = new OSSpecificInfo();
-            osSpecificInfo.setOs(osSpecificDTO.getOs());
-            osSpecificInfo.setArch(osSpecificDTO.getArch());
-            osSpecificInfo.setPackages(osSpecificDTO.getPackages());
-            osSpecificInfos.add(osSpecificInfo);
+        List<PackageSpecificInfo> packageSpecificInfos = new ArrayList<>();
+        for (PackageSpecificDTO packageSpecificDTO : packageSpecificDTOList) {
+            PackageSpecificInfo packageSpecificInfo = new PackageSpecificInfo();
+            packageSpecificInfo.setOs(packageSpecificDTO.getOs());
+            packageSpecificInfo.setArch(packageSpecificDTO.getArch());
+            List<PackageInfo> packageInfoList = new ArrayList<>();
+            for (PackageDTO packageDTO : packageSpecificDTO.getPackages()) {
+                PackageInfo packageInfo = new PackageInfo();
+                packageInfo.setName(packageDTO.getName());
+                packageInfo.setChecksum(packageDTO.getChecksum());
+                packageInfoList.add(packageInfo);
+            }
+            packageSpecificInfo.setPackages(packageInfoList);
+            packageSpecificInfos.add(packageSpecificInfo);
         }
 
-        return osSpecificInfos;
+        return packageSpecificInfos;
     }
 
     private List<CustomCommandInfo> convertCustomCommandInfo(List<CustomCommandDTO> customCommandDTOs) {

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java

@@ -71,17 +71,23 @@ public void beforeRun() {
 
     @Override
     public Boolean run() {
-        beforeRun();
+        boolean taskSuccess;
 
-        CommandRequest.Builder builder = CommandRequest.newBuilder(getCommandRequest());
-        builder.setTaskId(getTaskPO().getId());
-        commandRequest = builder.build();
+        try {
+            beforeRun();
 
-        CommandServiceGrpc.CommandServiceBlockingStub stub = GrpcClient.getBlockingStub(
-                taskContext.getHostname(), CommandServiceGrpc.CommandServiceBlockingStub.class);
-        CommandReply reply = stub.exec(commandRequest);
+            CommandRequest.Builder builder = CommandRequest.newBuilder(getCommandRequest());
+            builder.setTaskId(getTaskPO().getId());
+            commandRequest = builder.build();
 
-        boolean taskSuccess = reply != null && reply.getCode() == MessageConstants.SUCCESS_CODE;
+            CommandServiceGrpc.CommandServiceBlockingStub stub = GrpcClient.getBlockingStub(
+                    taskContext.getHostname(), CommandServiceGrpc.CommandServiceBlockingStub.class);
+            CommandReply reply = stub.exec(commandRequest);
+
+            taskSuccess = reply != null && reply.getCode() == MessageConstants.SUCCESS_CODE;
+        } catch (Exception e) {
+            taskSuccess = false;
+        }
 
         if (taskSuccess) {
             onSuccess();

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ServiceConverter.java

@@ -41,7 +41,7 @@ public interface ServiceConverter {
 
     ServiceConverter INSTANCE = Mappers.getMapper(ServiceConverter.class);
 
-    @Mapping(target = "osSpecifics", source = "osSpecifics", qualifiedByName = "obj2Json")
+    @Mapping(target = "packageSpecifics", source = "packageSpecifics", qualifiedByName = "obj2Json")
     @Mapping(target = "requiredServices", source = "requiredServices", qualifiedByName = "obj2Json")
     @Mapping(target = "clusterPO", expression = "java(clusterPO)")
     ServicePO fromDTO2PO(ServiceDTO serviceDTO, @Context ClusterPO clusterPO);

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageDTO.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bigtop.manager.server.model.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class PackageDTO {
+
+    private String name;
+
+    private String checksum;
+}

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/OSSpecificDTO.java → bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/dto/PackageSpecificDTO.java

@@ -23,11 +23,11 @@
 import java.util.List;
 
 @Data
-public class OSSpecificDTO {
+public class PackageSpecificDTO {
 
     private List<String> os;
 
     private List<String> arch;
 
-    private List<String> packages;
+    private List<PackageDTO> packages;
 }