BIGTOP-4319: Add clusterHosts info to infra services (apache#141)

bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java

@@ -27,6 +27,7 @@
 import lombok.ToString;
 
 import java.util.List;
+import java.util.Map;
 
 @Data
 @ToString(callSuper = true)
@@ -49,4 +50,10 @@ public class CommandPayload extends BasePayload {
     private String componentName;
 
     private List<PackageSpecificInfo> packageSpecifics;
+
+    /**
+     * This field is exclusively used for Prometheus and Grafana within the infra services.
+     * Includes cluster and corresponding hostname.
+     */
+    private Map<String, List<String>> clusterHosts;
 }

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java

@@ -108,6 +108,9 @@ public class HostPO extends BasePO implements Serializable {
     @Transient
     private String clusterName;
 
+    @Transient
+    private String clusterDisplayName;
+
     @Transient
     private Integer componentNum;
 }

bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml

@@ -42,7 +42,7 @@
         select
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
-        </include>, c.display_name as cluster_name, count(comp.id) as component_num
+        </include>, c.name as cluster_name, c.display_name as cluster_display_name, count(comp.id) as component_num
         from host h
         left join cluster c on h.cluster_id = c.id
         left join component comp on h.id = comp.host_id
@@ -67,7 +67,7 @@
         select
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
-        </include>, c.display_name as cluster_name, count(comp.id) as component_num
+        </include>, c.name as cluster_name, c.display_name as cluster_display_name, count(comp.id) as component_num
         from host h
         left join cluster c on h.cluster_id = c.id
         left join component comp on h.id = comp.host_id
@@ -95,7 +95,7 @@
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
         </include>
-        ,clus.display_name as cluster_name
+        , clus.name as cluster_name, clus.display_name as cluster_display_name
         from host h
         left join cluster clus
         on h.cluster_id = clus.id

bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml

@@ -42,7 +42,7 @@
         select
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
-        </include>, c.display_name as cluster_name, count(comp.id) as component_num
+        </include>, c.name as cluster_name, c.display_name as cluster_display_name, count(comp.id) as component_num
         from host h
         left join cluster c on h.cluster_id = c.id
         left join component comp on h.id = comp.host_id
@@ -67,7 +67,7 @@
         select
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
-        </include>, c.display_name as cluster_name, count(comp.id) as component_num
+        </include>, c.name as cluster_name, c.display_name as cluster_display_name, count(comp.id) as component_num
         from host h
         left join cluster c on h.cluster_id = c.id
         left join component comp on h.id = comp.host_id
@@ -95,7 +95,7 @@
         <include refid="baseColumnsV2">
             <property name="alias" value="h"/>
         </include>
-        ,clus.display_name as cluster_name
+        , clus.name as cluster_name, clus.display_name as cluster_display_name
         from
         host h
         left join cluster clus

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java

@@ -19,13 +19,16 @@
 package org.apache.bigtop.manager.server.command.stage;
 
 import org.apache.bigtop.manager.dao.po.ClusterPO;
+import org.apache.bigtop.manager.dao.po.HostPO;
 import org.apache.bigtop.manager.dao.repository.ClusterDao;
 import org.apache.bigtop.manager.server.command.task.TaskContext;
 import org.apache.bigtop.manager.server.holder.SpringContextHolder;
 import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
 import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public abstract class AbstractComponentStage extends AbstractStage {
@@ -77,7 +80,21 @@ protected TaskContext createTaskContext(String hostname) {
 
         Map<String, Object> properties = new HashMap<>();
         properties.put("packageSpecifics", serviceDTO.getPackageSpecifics());
+        properties.put("clusterHosts", getClusterHosts());
         taskContext.setProperties(properties);
         return taskContext;
     }
+
+    private Map<String, List<String>> getClusterHosts() {
+        Map<String, List<String>> clusterHosts = new HashMap<>();
+        for (ClusterPO clusterPO : clusterDao.findAll()) {
+            List<String> hosts = new ArrayList<>();
+            for (HostPO hostPO : hostDao.findAllByClusterId(clusterPO.getId())) {
+                String host = hostPO.getHostname();
+                hosts.add(host);
+            }
+            clusterHosts.put(clusterPO.getName(), hosts);
+        }
+        return clusterHosts;
+    }
 }

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java

@@ -67,7 +67,9 @@ protected CommandRequest getCommandRequest() {
 
         commandPayload.setPackageSpecifics(
                 convertPackageSpecificInfo((List<PackageSpecificDTO>) properties.get("packageSpecifics")));
-
+        if (stackDTO.getStackName().equals("infra")) {
+            commandPayload.setClusterHosts((Map<String, List<String>>) properties.get("clusterHosts"));
+        }
         CommandRequest.Builder builder = CommandRequest.newBuilder();
         builder.setType(CommandType.COMPONENT);
         builder.setPayload(JsonUtils.writeAsString(commandPayload));

bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/HostVO.java

@@ -71,5 +71,7 @@ public class HostVO {
 
     private String clusterName;
 
+    private String clusterDisplayName;
+
     private Integer componentNum;
 }

bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java

@@ -28,6 +28,8 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.io.File;
+import java.util.List;
+import java.util.Map;
 
 @Slf4j
 @NoArgsConstructor
@@ -72,4 +74,9 @@ public String stackHome() {
 
         return parentPath + "/infras";
     }
+
+    public Map<String, List<String>> getClusterHosts() {
+        // In Component Status stage, clusterHosts is null
+        return commandPayload.getClusterHosts();
+    }
 }

bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java

@@ -44,6 +44,7 @@ public class PrometheusParams extends InfraParams {
     protected final String PROMETHEUS_SELF_JOB_NAME = "prometheus";
     protected final String BM_AGENT_JOB_NAME = "bm-agent";
     protected final String BM_AGENT_PORT = "8081";
+    protected final String AGENT_TARGET_LABEL = "cluster";
 
     private Map<String, Object> prometheusScrapeJob;
     private Map<String, Object> agentScrapeJob;
@@ -55,6 +56,7 @@ public class PrometheusParams extends InfraParams {
 
     public PrometheusParams(CommandPayload commandPayload) {
         super(commandPayload);
+        setAgentScrapeJob();
         scrapeJobs = new ArrayList<>();
         scrapeJobs.add(prometheusScrapeJob);
         scrapeJobs.add(agentScrapeJob);
@@ -75,38 +77,22 @@ public String getServiceName() {
         return "prometheus";
     }
 
-    protected List<String> getAllHost() {
-        List<String> ips = LocalSettings.hosts().get("all");
-        List<String> hosts = new ArrayList<>();
-        for (String ip : ips) {
-            hosts.add(MessageFormat.format("{0}:{1}", ip, BM_AGENT_PORT));
-        }
-        return hosts;
-    }
-
     @GlobalParams
     public Map<String, Object> prometheusJob() {
         Map<String, Object> configuration = LocalSettings.configurations(getServiceName(), "prometheus");
         prometheusPort = (String) configuration.get("port");
         Map<String, Object> job = new HashMap<>();
         job.put("name", PROMETHEUS_SELF_JOB_NAME);
         job.put("targets_file", targetsConfigFile(PROMETHEUS_SELF_JOB_NAME));
-        job.put("targets_list", List.of(MessageFormat.format("localhost:{0}", prometheusPort)));
+
+        Map<String, Object> target = new HashMap<>();
+        target.put("targets", List.of(MessageFormat.format("localhost:{0}", prometheusPort)));
+        job.put("targets_list", List.of(target));
+
         prometheusScrapeJob = job;
         return configuration;
     }
 
-    @GlobalParams
-    public Map<String, Object> agentJob() {
-        Map<String, Object> job = new HashMap<>();
-        job.put("name", BM_AGENT_JOB_NAME);
-        job.put("targets_file", targetsConfigFile(BM_AGENT_JOB_NAME));
-        job.put("targets_list", getAllHost());
-        job.put("metrics_path", "/actuator/prometheus");
-        agentScrapeJob = job;
-        return LocalSettings.configurations(getServiceName(), "prometheus");
-    }
-
     @GlobalParams
     public Map<String, Object> configs() {
         Map<String, Object> configuration = LocalSettings.configurations(getServiceName(), "prometheus");
@@ -127,4 +113,28 @@ public Map<String, Object> rules() {
     public String listenAddress() {
         return MessageFormat.format("0.0.0.0:{0}", prometheusPort);
     }
+
+    public void setAgentScrapeJob() {
+        agentScrapeJob = new HashMap<>();
+        agentScrapeJob.put("name", BM_AGENT_JOB_NAME);
+        agentScrapeJob.put("targets_file", targetsConfigFile(BM_AGENT_JOB_NAME));
+        agentScrapeJob.put("metrics_path", "/actuator/prometheus");
+
+        List<Map<String, Object>> agentTargets = new ArrayList<>();
+        Map<String, List<String>> clusterHosts = getClusterHosts();
+        if (clusterHosts != null) {
+            clusterHosts.forEach((cluster, hosts) -> {
+                Map<String, Object> agentTarget = new HashMap<>();
+                List<String> targets = new ArrayList<>();
+                for (String host : hosts) {
+                    targets.add(MessageFormat.format("{0}:{1}", host, BM_AGENT_PORT));
+                }
+                agentTarget.put("targets", targets);
+                agentTarget.put("labels", Map.of(AGENT_TARGET_LABEL, cluster));
+                agentTargets.add(agentTarget);
+            });
+        }
+
+        agentScrapeJob.put("targets_list", agentTargets);
+    }
 }

bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java

@@ -29,15 +29,12 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 @Slf4j
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class PrometheusSetup {
 
-    @SuppressWarnings("unchecked")
     public static ShellResult config(Params params) {
         PrometheusParams prometheusParams = (PrometheusParams) params;
         String user = prometheusParams.user();
@@ -65,16 +62,15 @@ public static ShellResult config(Params params) {
 
         for (int i = 0; i < prometheusParams.getScrapeJobs().size(); i++) {
             Map<String, Object> job = prometheusParams.getScrapeJobs().get(i);
-            Map<String, List<String>> targets = new HashMap<>();
-            targets.put("targets", (List<String>) job.get("targets_list"));
             LinuxFileUtils.toFile(
                     ConfigType.JSON,
                     (String) job.get("targets_file"),
                     user,
                     group,
                     Constants.PERMISSION_644,
-                    List.of(targets));
+                    job.get("targets_list"));
         }
+
         return ShellResult.success("Prometheus Configure success!");
     }
 }