add cluster hosts

lhpqaq · Jan 4, 2025 · 01ce16f · 01ce16f
1 parent a07bc6d
commit 01ce16f
Show file tree

Hide file tree

Showing 6 changed files with 88 additions and 8 deletions.
diff --git a/...src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java b/...src/main/java/org/apache/bigtop/manager/common/message/entity/payload/CommandPayload.java
@@ -27,6 +27,7 @@
 import lombok.ToString;
 
 import java.util.List;
+import java.util.Map;
 
 @Data
 @ToString(callSuper = true)
@@ -49,4 +50,10 @@ public class CommandPayload extends BasePayload {
     private String componentName;
 
     private List<PackageSpecificInfo> packageSpecifics;
+
+    /**
+     * This field is exclusively used for Prometheus and Grafana within the infra services.
+     * Includes cluster and corresponding host IP.
+     */
+    private Map<String, List<String>> clusterHosts;
 }
diff --git a/.../src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java b/.../src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractComponentStage.java
@@ -19,13 +19,15 @@
 package org.apache.bigtop.manager.server.command.stage;
 
 import org.apache.bigtop.manager.dao.po.ClusterPO;
+import org.apache.bigtop.manager.dao.po.HostPO;
 import org.apache.bigtop.manager.dao.repository.ClusterDao;
 import org.apache.bigtop.manager.server.command.task.TaskContext;
 import org.apache.bigtop.manager.server.holder.SpringContextHolder;
 import org.apache.bigtop.manager.server.model.dto.ComponentDTO;
 import org.apache.bigtop.manager.server.model.dto.ServiceDTO;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public abstract class AbstractComponentStage extends AbstractStage {
@@ -77,7 +79,23 @@ protected TaskContext createTaskContext(String hostname) {
 
         Map<String, Object> properties = new HashMap<>();
         properties.put("packageSpecifics", serviceDTO.getPackageSpecifics());
+        properties.put("clusterHosts", getClusterHosts());
         taskContext.setProperties(properties);
         return taskContext;
     }
+
+    private Map<String, List<String>> getClusterHosts() {
+        Map<String, List<String>> clusterHosts = new HashMap<>();
+        List<HostPO> hostPOs = hostDao.findAll();
+        for (HostPO hostPO : hostPOs) {
+            String clusterName = hostPO.getClusterName();
+            String host = hostPO.getIpv4();
+            if (clusterHosts.containsKey(clusterName)) {
+                clusterHosts.get(clusterName).add(host);
+            } else {
+                clusterHosts.put(clusterName, List.of(host));
+            }
+        }
+        return clusterHosts;
+    }
 }
diff --git a/...er/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java b/...er/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractComponentTask.java
@@ -67,6 +67,9 @@ protected CommandRequest getCommandRequest() {
 
         commandPayload.setPackageSpecifics(
                 convertPackageSpecificInfo((List<PackageSpecificDTO>) properties.get("packageSpecifics")));
+        if (stackDTO.getStackName().equals("infra")) {
+            commandPayload.setClusterHosts((Map<String, List<String>>) properties.get("clusterHosts"));
+        }
 
         CommandRequest.Builder builder = CommandRequest.newBuilder();
         builder.setType(CommandType.COMPONENT);

diff --git a/...er-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java b/...er-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java
@@ -28,13 +28,18 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.io.File;
+import java.util.List;
+import java.util.Map;
 
 @Slf4j
 @NoArgsConstructor
 public abstract class InfraParams extends BaseParams {
 
+    protected Map<String, List<String>> clusterHosts;
+
     protected InfraParams(CommandPayload commandPayload) {
         super(commandPayload);
+        clusterHosts = commandPayload.getClusterHosts();
     }
 
     /**
@@ -72,4 +77,11 @@ public String stackHome() {
 
         return parentPath + "/infras";
     }
+
+    public Map<String, List<String>> getClusterHosts() {
+        if (clusterHosts == null) {
+            log.error("Cluster hosts is null, please check the command payload");
+        }
+        return clusterHosts;
+    }
 }
diff --git a/...c/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java b/...c/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java
@@ -44,6 +44,7 @@ public class PrometheusParams extends InfraParams {
     protected final String PROMETHEUS_SELF_JOB_NAME = "prometheus";
     protected final String BM_AGENT_JOB_NAME = "bm-agent";
     protected final String BM_AGENT_PORT = "8081";
+    protected final String AGENT_TARGET_LABEL = "cluster";
 
     private Map<String, Object> prometheusScrapeJob;
     private Map<String, Object> agentScrapeJob;
@@ -91,7 +92,11 @@ public Map<String, Object> prometheusJob() {
         Map<String, Object> job = new HashMap<>();
         job.put("name", PROMETHEUS_SELF_JOB_NAME);
         job.put("targets_file", targetsConfigFile(PROMETHEUS_SELF_JOB_NAME));
-        job.put("targets_list", List.of(MessageFormat.format("localhost:{0}", prometheusPort)));
+
+        Map<String, Object> target = new HashMap<>();
+        target.put("targets", List.of(MessageFormat.format("localhost:{0}", prometheusPort)));
+        job.put("targets_list", List.of(target));
+
         prometheusScrapeJob = job;
         return configuration;
     }
@@ -101,8 +106,17 @@ public Map<String, Object> agentJob() {
         Map<String, Object> job = new HashMap<>();
         job.put("name", BM_AGENT_JOB_NAME);
         job.put("targets_file", targetsConfigFile(BM_AGENT_JOB_NAME));
-        job.put("targets_list", getAllHost());
         job.put("metrics_path", "/actuator/prometheus");
+
+        List<Map<String, Object>> agentTargets = new ArrayList<>();
+        getClusterHosts().forEach((cluster, hosts) -> {
+            Map<String, Object> agentTarget = new HashMap<>();
+            agentTarget.put("targets", hosts);
+            agentTarget.put("labels", Map.of(AGENT_TARGET_LABEL, cluster));
+            agentTargets.add(agentTarget);
+        });
+        job.put("targets_list", agentTargets);
+
         agentScrapeJob = job;
         return LocalSettings.configurations(getServiceName(), "prometheus");
     }

diff --git a/...rc/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java b/...rc/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java
@@ -29,15 +29,15 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 @Slf4j
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class PrometheusSetup {
 
-    @SuppressWarnings("unchecked")
+    //    private static final String AGENT_TARGET_LABEL = "cluster";
+
+    //    @SuppressWarnings("unchecked")
     public static ShellResult config(Params params) {
         PrometheusParams prometheusParams = (PrometheusParams) params;
         String user = prometheusParams.user();
@@ -63,18 +63,44 @@ public static ShellResult config(Params params) {
                 Constants.PERMISSION_644,
                 prometheusParams.getGlobalParamsMap());
 
+        //        Map<String, List<String>> prometheusTargets = new HashMap<>();
+        //        prometheusTargets.put("targets", (List<String>)
+        // prometheusParams.getPrometheusScrapeJob().get("targets_list"));
+        //        LinuxFileUtils.toFile(
+        //                ConfigType.JSON,
+        //                (String) prometheusParams.getPrometheusScrapeJob().get("targets_file"),
+        //                user,
+        //                group,
+        //                Constants.PERMISSION_644,
+        //                List.of(prometheusTargets));
+        //
+        //        List<Map<String, Object>> agentTargets = new ArrayList<>();
+        //        prometheusParams.getClusterHosts().forEach((cluster, hosts) -> {
+        //            Map<String, Object> agentTarget = new HashMap<>();
+        //            agentTarget.put("targets", hosts);
+        //            agentTarget.put("labels", Map.of(AGENT_TARGET_LABEL, cluster));
+        //            agentTargets.add(agentTarget);
+        //        });
+        //
+        //        LinuxFileUtils.toFile(
+        //                ConfigType.JSON,
+        //                (String) prometheusParams.getAgentScrapeJob().get("targets_file"),
+        //                user,
+        //                group,
+        //                Constants.PERMISSION_644,
+        //                agentTargets);
+
         for (int i = 0; i < prometheusParams.getScrapeJobs().size(); i++) {
             Map<String, Object> job = prometheusParams.getScrapeJobs().get(i);
-            Map<String, List<String>> targets = new HashMap<>();
-            targets.put("targets", (List<String>) job.get("targets_list"));
             LinuxFileUtils.toFile(
                     ConfigType.JSON,
                     (String) job.get("targets_file"),
                     user,
                     group,
                     Constants.PERMISSION_644,
-                    List.of(targets));
+                    job.get("targets_list"));
         }
+
         return ShellResult.success("Prometheus Configure success!");
     }
 }