diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..58129053 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.classpath +.settings +.project diff --git a/kafka-eagle-api/.gitignore b/kafka-eagle-api/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/kafka-eagle-api/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/kafka-eagle-common/.gitignore b/kafka-eagle-common/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/kafka-eagle-common/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/kafka-eagle-common/src/main/resources/system-config.properties b/kafka-eagle-common/src/main/resources/system-config.properties index a6d199d8..603313c5 100644 --- a/kafka-eagle-common/src/main/resources/system-config.properties +++ b/kafka-eagle-common/src/main/resources/system-config.properties @@ -2,31 +2,23 @@ # multi zookeeper & kafka cluster list ###################################### kafka.eagle.zk.cluster.alias=cluster1 -cluster1.zk.list=127.0.0.1:2181 -cluster1_0_1.zk.list=127.0.0.1:2181/v_1_0_1 -cluster2.zk.list=127.0.0.1:2181/plain -cluster3.zk.list=127.0.0.1:2181/scram -cluster4.zk.list=127.0.0.1:2182 +cluster1.zk.list=10.100.172.147:2181 -###################################### # zookeeper enable acl ###################################### -cluster4.zk.acl.enable=false -cluster4.zk.acl.schema=digest -cluster4.zk.acl.username=test -cluster4.zk.acl.password=test123 - +cluster1.zk.acl.enable=false +cluster1.zk.acl.schema=digest +cluster1.zk.acl.username=kafka +cluster1.zk.acl.password=kafkapasswd ###################################### # broker size online list ###################################### cluster1.kafka.eagle.broker.size=10 -cluster1_0_1.kafka.eagle.broker.size=10 -cluster4.kafka.eagle.broker.size=10 ###################################### # zk client thread limit ###################################### -kafka.zk.limit.size=32 +kafka.zk.limit.size=1 ###################################### # kafka eagle webui port @@ -47,10 +39,7 @@ cluster1.kafka.eagle.jmx.truststore.password=ke123456 # kafka offset storage ###################################### cluster1.kafka.eagle.offset.storage=kafka -cluster1_0_1.kafka.eagle.offset.storage=kafka -cluster2.kafka.eagle.offset.storage=kafka -#cluster3.kafka.eagle.offset.storage=kafka -cluster4.kafka.eagle.offset.storage=kafka + ###################################### # kafka metrics, 15 days by default @@ -83,14 +72,15 @@ kafka.eagle.topic.token=keadmin ###################################### # kafka sasl authenticate ###################################### -cluster1.kafka.eagle.sasl.enable=false +cluster1.kafka.eagle.sasl.enable=true cluster1.kafka.eagle.sasl.protocol=SASL_PLAINTEXT cluster1.kafka.eagle.sasl.mechanism=SCRAM-SHA-256 -cluster1.kafka.eagle.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin-secret"; +cluster1.kafka.eagle.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafkaadmin" password="kafkaadmin"; cluster1.kafka.eagle.sasl.client.id= cluster1.kafka.eagle.blacklist.topics= cluster1.kafka.eagle.sasl.cgroup.enable=false cluster1.kafka.eagle.sasl.cgroup.topics=kafka20191230,kafka20191217 + cluster4.kafka.eagle.sasl.enable=false cluster4.kafka.eagle.sasl.protocol=SASL_PLAINTEXT cluster4.kafka.eagle.sasl.mechanism=PLAIN @@ -113,11 +103,12 @@ cluster1.kafka.eagle.ssl.cgroup.topics= ###################################### # kafka jdbc driver address ###################################### -kafka.eagle.driver=com.mysql.jdbc.Driver -kafka.eagle.url=jdbc:mysql://127.0.0.1:3306/ke?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull -kafka.eagle.username=root -kafka.eagle.password=123456 -#kafka.eagle.driver=org.sqlite.JDBC -#kafka.eagle.url=jdbc:sqlite:/Users/dengjie/webserver/hadoop/sqlite/ke.db +#kafka.eagle.driver=com.mysql.jdbc.Driver +#kafka.eagle.url=jdbc:mysql://127.0.0.1:3306/ke?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull #kafka.eagle.username=root -#kafka.eagle.password=root +#kafka.eagle.password=123456 + +kafka.eagle.driver=org.sqlite.JDBC +kafka.eagle.url=jdbc:sqlite:/Users/jefffeng/kenew.db +kafka.eagle.username=root +kafka.eagle.password=root diff --git a/kafka-eagle-common/src/main/resources/system-config.properties.orig b/kafka-eagle-common/src/main/resources/system-config.properties.orig new file mode 100644 index 00000000..df10244c --- /dev/null +++ b/kafka-eagle-common/src/main/resources/system-config.properties.orig @@ -0,0 +1,139 @@ +###################################### +# multi zookeeper & kafka cluster list +###################################### +kafka.eagle.zk.cluster.alias=cluster1 +<<<<<<< HEAD +cluster1.zk.list=10.100.172.147:2181,10.100.172.171:2181,10.100.172.173:2181 +======= +cluster1.zk.list=127.0.0.1:2181 +cluster1_0_1.zk.list=127.0.0.1:2181/v_1_0_1 +cluster2.zk.list=127.0.0.1:2181/plain +cluster3.zk.list=127.0.0.1:2181/scram +cluster4.zk.list=127.0.0.1:2182 + +###################################### +# zookeeper enable acl +###################################### +cluster4.zk.acl.enable=false +cluster4.zk.acl.schema=digest +cluster4.zk.acl.username=test +cluster4.zk.acl.password=test123 +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 + +###################################### +# broker size online list +###################################### +cluster1.kafka.eagle.broker.size=10 + +###################################### +# zk client thread limit +###################################### +kafka.zk.limit.size=32 + +###################################### +# kafka eagle webui port +###################################### +kafka.eagle.webui.port=8048 + +###################################### +# kafka jmx acl and ssl authenticate +###################################### +cluster1.kafka.eagle.jmx.acl=false +cluster1.kafka.eagle.jmx.user=keadmin +cluster1.kafka.eagle.jmx.password=keadmin123 +cluster1.kafka.eagle.jmx.ssl=false +cluster1.kafka.eagle.jmx.truststore.location=/Users/dengjie/workspace/ssl/certificates/kafka.truststore +cluster1.kafka.eagle.jmx.truststore.password=ke123456 + +###################################### +# kafka offset storage +###################################### +cluster1.kafka.eagle.offset.storage=kafka + + +###################################### +# kafka metrics, 15 days by default +###################################### +kafka.eagle.metrics.charts=true +kafka.eagle.metrics.retain=15 + +###################################### +# master quartz time +###################################### +kafka.eagle.quartz.master.time=* */1 * * * ? + +###################################### +# kafka sql topic records max +###################################### +kafka.eagle.sql.topic.records.max=5000 +kafka.eagle.sql.worknode.port=8787 +kafka.eagle.sql.distributed.enable=false +kafka.eagle.sql.worknode.rpc.timeout=300000 +kafka.eagle.sql.worknode.fetch.threshold=5000 +kafka.eagle.sql.worknode.fetch.timeout=20000 +kafka.eagle.sql.worknode.server.path=/Users/dengjie/workspace/kafka-eagle-plus/kafka-eagle-common/src/main/resources/works + + +###################################### +# delete kafka topic token +###################################### +kafka.eagle.topic.token=keadmin + +###################################### +# kafka sasl authenticate +###################################### +cluster1.kafka.eagle.sasl.enable=false +cluster1.kafka.eagle.sasl.protocol=SASL_PLAINTEXT +cluster1.kafka.eagle.sasl.mechanism=SCRAM-SHA-256 +#cluster1.kafka.eagle.sasl.mechanism=PLAIN +cluster1.kafka.eagle.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafkaadmin" password="kafkaadmin"; +cluster1.kafka.eagle.sasl.client.id= +cluster1.kafka.eagle.blacklist.topics= +cluster1.kafka.eagle.sasl.cgroup.enable=false +<<<<<<< HEAD +cluster1.kafka.eagle.sasl.cgroup.topics= +======= +cluster1.kafka.eagle.sasl.cgroup.topics=kafka20191230,kafka20191217 +cluster4.kafka.eagle.sasl.enable=false +cluster4.kafka.eagle.sasl.protocol=SASL_PLAINTEXT +cluster4.kafka.eagle.sasl.mechanism=PLAIN +cluster4.kafka.eagle.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret"; +cluster4.kafka.eagle.sasl.client.id= +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 + +###################################### +# kafka ssl authenticate +###################################### +cluster1.kafka.eagle.ssl.enable=false +cluster1.kafka.eagle.ssl.protocol=SSL +cluster1.kafka.eagle.ssl.truststore.location=/Users/dengjie/workspace/ssl/certificates/kafka.truststore +cluster1.kafka.eagle.ssl.truststore.password=ke123456 +cluster1.kafka.eagle.ssl.keystore.location=/Users/dengjie/workspace/ssl/certificates/kafka.keystore +cluster1.kafka.eagle.ssl.keystore.password=ke123456 +cluster1.kafka.eagle.ssl.key.password=ke123456 +cluster1.kafka.eagle.ssl.cgroup.enable=false +cluster1.kafka.eagle.ssl.cgroup.topics= + +###################################### +# kafka jdbc driver address +###################################### +<<<<<<< HEAD +#kafka.eagle.driver=com.mysql.jdbc.Driver +#kafka.eagle.url=jdbc:mysql://10.100.172.33:3306/ke?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull +#kafka.eagle.username=dev +#kafka.eagle.password=123456 + +kafka.eagle.driver=org.sqlite.JDBC +kafka.eagle.url=jdbc:sqlite:/Users/jefffeng/ke.db +kafka.eagle.username=root +kafka.eagle.password=root +======= +kafka.eagle.driver=com.mysql.jdbc.Driver +kafka.eagle.url=jdbc:mysql://127.0.0.1:3306/ke?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull +kafka.eagle.username=root +kafka.eagle.password=123456 +#kafka.eagle.driver=org.sqlite.JDBC +#kafka.eagle.url=jdbc:sqlite:/Users/dengjie/webserver/hadoop/sqlite/ke.db +#kafka.eagle.username=root +#kafka.eagle.password=root +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 diff --git a/kafka-eagle-core/.gitignore b/kafka-eagle-core/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/kafka-eagle-core/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java index 83c1f286..8f21edae 100644 --- a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java @@ -20,6 +20,9 @@ import org.apache.kafka.common.TopicPartition; import org.smartloli.kafka.eagle.common.protocol.*; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + import java.util.List; import java.util.Map; import java.util.Properties; @@ -91,10 +94,8 @@ public interface KafkaService { */ public Map getKafkaOffset(String clusterAlias, String group, String topic, Set partitionids); - /** - * Use kafka console comand to create topic. - */ - public Map create(String clusterAlias, String topicName, String partitions, String replic); + /** Use kafka console comand to create topic. */ + public Map create(String clusterAlias, String topicName, String partitions, String replic, String username); /** * Use kafka console command to delete topic. @@ -259,5 +260,17 @@ public interface KafkaService { * Set kafka ssl acl. */ public void ssl(Properties props, String clusterAlias); + + /** + * Get kafka acls . + */ + public JSONArray getKafkaAcl(String clusterAlias); + + public JSONArray getKafkaAclBYTopicName(String clusterAlias, String topicname); + + Map createGroup(String clusterAlias, String userName, String groupName); + + String deleteAcl(String clusterAlias, JSONObject jsonObject); + } diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java.orig b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java.orig new file mode 100644 index 00000000..c975bb43 --- /dev/null +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaService.java.orig @@ -0,0 +1,389 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.core.factory; + +import org.apache.kafka.common.TopicPartition; +import org.smartloli.kafka.eagle.common.protocol.*; + +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * Kafka group,topic and partition interface. + * + * @author smartloli. + *

+ * Created by Jan 18, 2017 + *

+ * Update by hexiang 20170216 + */ +public interface KafkaService { + +<<<<<<< HEAD + /** Find topic and group exist in zookeeper. */ + public boolean findTopicAndGroupExist(String clusterAlias, String topic, String group); + + /** Obtaining metadata in zookeeper by topic. */ + public List findTopicPartition(String clusterAlias, String topic); + + /** Get kafka active consumer topic. */ + public Map> getActiveTopic(String clusterAlias); + + /** Get kafka active consumer topic. */ + public Set getActiveTopic(String clusterAlias, String group); + + /** Get all broker list from zookeeper. */ + public List getAllBrokersInfo(String clusterAlias); + + /** Get broker host info from ids. */ + public String getBrokerJMXFromIds(String clusterAlias, int ids); + + /** Obtaining kafka consumer information from zookeeper. */ + public Map> getConsumers(String clusterAlias); + + /** Obtaining kafka consumer page information from zookeeper. */ + public Map> getConsumers(String clusterAlias, DisplayInfo page); + + /** According to group, topic and partition to get offset from zookeeper. */ + public OffsetZkInfo getOffset(String clusterAlias, String topic, String group, int partition); + + /** Get kafka 0.10.x offset from topic. */ + public String getKafkaOffset(String clusterAlias); + + /** Get the data for the topic partition in the specified consumer group */ + public Map getKafkaOffset(String clusterAlias, String group, String topic, Set partitionids); + + /** Use kafka console comand to create topic. */ + public Map create(String clusterAlias, String topicName, String partitions, String replic, String username); + + /** Use kafka console command to delete topic. */ + public Map delete(String clusterAlias, String topicName); + + /** Convert query kafka to topic in the sql message for standard sql. */ + public KafkaSqlInfo parseSql(String clusterAlias, String sql); + + /** Get kafka 0.10.x active consumer group & topics. */ + public Set getKafkaActiverTopics(String clusterAlias, String group); + + /** Get kafka 0.10.x consumer topic, maybe consumer topic owner is null. */ + public Set getKafkaConsumerTopics(String clusterAlias, String group); + + /** Get kafka 0.10.x consumer group & topic information. */ + public String getKafkaConsumer(String clusterAlias); + + /** Get kafka 0.10.x consumer group & topic information used for page. */ + public String getKafkaConsumer(String clusterAlias, DisplayInfo page); + + @Deprecated + /** Get kafka consumer information pages. */ + public String getKafkaActiverSize(String clusterAlias, String group); + + /** Get kafka consumer information pages not owners. */ + public OwnerInfo getKafkaActiverNotOwners(String clusterAlias, String group); + + /** Get kafka broker bootstrap server. */ + public String getKafkaBrokerServer(String clusterAlias); + + /** Get kafka consumer groups. */ + public int getKafkaConsumerGroups(String clusterAlias); + + /** Get kafka consumer topics. */ + public Set getKafkaConsumerTopic(String clusterAlias, String group); + + /** Get kafka consumer group & topic. */ + public String getKafkaConsumerGroupTopic(String clusterAlias, String group); + + /** Get kafka topic history logsize . */ + public long getKafkaLogSize(String clusterAlias, String topic, int partitionid); + + /** Get kafka topic history batch logsize. */ + public Map getKafkaLogSize(String clusterAlias, String topic, Set partitionids); + + /** Get kafka topic real logsize by partitionid. */ + public long getKafkaRealLogSize(String clusterAlias, String topic, int partitionid); + + /** Get kafka topic real logsize by partitionid set. */ + public long getKafkaRealLogSize(String clusterAlias, String topic, Set partitionids); + + /** Get topic producer send logsize records. */ + public long getKafkaProducerLogSize(String clusterAlias, String topic, Set partitionids); + + /** Get kafka sasl topic metadate. */ + public List findKafkaLeader(String clusterAlias, String topic); + + /** Send mock message to kafka. */ + public boolean mockMessage(String clusterAlias, String topic, String message); + + /** Get kafka consumer group all topics lag. */ + public long getKafkaLag(String clusterAlias, String group, String topic); + + /** Get consumer group all topics lag. */ + public long getLag(String clusterAlias, String group, String topic); + + /** Get kafka history logsize by old version. */ + public long getLogSize(String clusterAlias, String topic, int partitionid); + + /** Get kafka history logsize by old version. */ + public long getLogSize(String clusterAlias, String topic, Set partitionids); + + /** Get kafka real logsize by old version partition set. */ + public long getRealLogSize(String clusterAlias, String topic, int partitionid); + + /** Get topic metadata. */ + public String getReplicasIsr(String clusterAlias, String topic, int partitionid); + + /** Get kafka version. */ + public String getKafkaVersion(String host, int port, String ids, String clusterAlias); + + /** Get kafka os memory. */ + public long getOSMemory(String host, int port, String property); + + /** Set kafka sasl acl. */ + public void sasl(Properties props, String clusterAlias); + + /** Set kafka ssl acl. */ + public void ssl(Properties props, String clusterAlias); +======= + /** + * Find topic and group exist in zookeeper. + */ + public boolean findTopicAndGroupExist(String clusterAlias, String topic, String group); + + /** + * Obtaining metadata in zookeeper by topic. + */ + public List findTopicPartition(String clusterAlias, String topic); + + /** + * Get kafka active consumer topic. + */ + public Map> getActiveTopic(String clusterAlias); + + /** + * Get kafka active consumer topic. + */ + public Set getActiveTopic(String clusterAlias, String group); + + /** + * Get all broker list from zookeeper. + */ + public List getAllBrokersInfo(String clusterAlias); + + /** + * Get broker host info from ids. + */ + public String getBrokerJMXFromIds(String clusterAlias, int ids); + + /** + * Obtaining kafka consumer information from zookeeper. + */ + public Map> getConsumers(String clusterAlias); + + /** + * Obtaining kafka consumer page information from zookeeper. + */ + public Map> getConsumers(String clusterAlias, DisplayInfo page); + + /** + * According to group, topic and partition to get offset from zookeeper. + */ + public OffsetZkInfo getOffset(String clusterAlias, String topic, String group, int partition); + + /** + * Get kafka 0.10.x offset from topic. + */ + public String getKafkaOffset(String clusterAlias); + + /** + * Get the data for the topic partition in the specified consumer group + */ + public Map getKafkaOffset(String clusterAlias, String group, String topic, Set partitionids); + + /** + * Use kafka console comand to create topic. + */ + public Map create(String clusterAlias, String topicName, String partitions, String replic); + + /** + * Use kafka console command to delete topic. + */ + public Map delete(String clusterAlias, String topicName); + + /** + * Convert query kafka to topic in the sql message for standard sql. + */ + public KafkaSqlInfo parseSql(String clusterAlias, String sql); + + /** + * Get kafka 0.10.x active consumer group & topics. + */ + public Set getKafkaActiverTopics(String clusterAlias, String group); + + /** + * Get kafka 0.10.x consumer topic, maybe consumer topic owner is null. + */ + public Set getKafkaConsumerTopics(String clusterAlias, String group); + + /** + * Get kafka 0.10.x consumer group & topic information. + */ + public String getKafkaConsumer(String clusterAlias); + + /** + * Get kafka 0.10.x consumer group & topic information used for page. + */ + public String getKafkaConsumer(String clusterAlias, DisplayInfo page); + + @Deprecated + /** Get kafka consumer information pages. */ + public String getKafkaActiverSize(String clusterAlias, String group); + + /** + * Get kafka consumer information pages not owners. + */ + public OwnerInfo getKafkaActiverNotOwners(String clusterAlias, String group); + + /** + * Get kafka broker bootstrap server. + */ + public String getKafkaBrokerServer(String clusterAlias); + + /** + * Get kafka consumer groups. + */ + public int getKafkaConsumerGroups(String clusterAlias); + + /** + * Get kafka consumer topics. + */ + public Set getKafkaConsumerTopic(String clusterAlias, String group); + + /** + * Get kafka consumer group & topic. + */ + public String getKafkaConsumerGroupTopic(String clusterAlias, String group); + + /** + * Get kafka topic history logsize . + */ + public long getKafkaLogSize(String clusterAlias, String topic, int partitionid); + + /** + * Get kafka topic history batch logsize. + */ + public Map getKafkaLogSize(String clusterAlias, String topic, Set partitionids); + + /** + * Get kafka topic real logsize by partitionid. + */ + public long getKafkaRealLogSize(String clusterAlias, String topic, int partitionid); + + /** + * Get kafka topic real logsize by partitionid set. + */ + public long getKafkaRealLogSize(String clusterAlias, String topic, Set partitionids); + + /** + * Get topic producer send logsize records. + */ + public long getKafkaProducerLogSize(String clusterAlias, String topic, Set partitionids); + + /** + * Get kafka sasl topic metadate. + */ + public List findKafkaLeader(String clusterAlias, String topic); + + /** + * Send mock message to kafka. + */ + public boolean mockMessage(String clusterAlias, String topic, String message); + + /** + * Get kafka consumer group all topics lag. + */ + public long getKafkaLag(String clusterAlias, String group, String topic); + + /** + * Get consumer group all topics lag. + */ + public long getLag(String clusterAlias, String group, String topic); + + /** + * Get kafka history logsize by old version. + */ + public long getLogSize(String clusterAlias, String topic, int partitionid); + + /** + * Get kafka history logsize by old version. + */ + public long getLogSize(String clusterAlias, String topic, Set partitionids); + + /** + * Get kafka real logsize by old version partition set. + */ + public long getRealLogSize(String clusterAlias, String topic, int partitionid); + + /** + * Get kafka real logsize by old version partition set. + */ + public long getRealLogSize(String clusterAlias, String topic, Set partitionids); + + /** + * Get topic metadata. + */ + public String getReplicasIsr(String clusterAlias, String topic, int partitionid); + + /** + * Get kafka version. + */ + public String getKafkaVersion(String host, int port, String ids, String clusterAlias); + + /** + * Get kafka os memory. + */ + public long getOSMemory(String clusterAlias, String host, int port, String property); + + /** + * Get kafka used cpu. + */ + public String getUsedCpu(String clusterAlias, String host, int port); + + /** + * Get kafka used cpu value. + */ + public double getUsedCpuValue(String clusterAlias, String host, int port); + + /** + * Get kafka used memory. + */ + public String getUsedMemory(String clusterAlias, String host, int port); + + /** + * Set kafka sasl acl. + */ + public void sasl(Properties props, String clusterAlias); + + /** + * Set kafka ssl acl. + */ + public void ssl(Properties props, String clusterAlias); +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 + +} diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java index 34795a04..4e9edb23 100644 --- a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java @@ -22,17 +22,38 @@ import com.alibaba.fastjson.JSONObject; import kafka.zk.KafkaZkClient; import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.admin.*; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ConsumerGroupListing; +import org.apache.kafka.clients.admin.CreateAclsResult; +import org.apache.kafka.clients.admin.DeleteAclsResult; +import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults; +import org.apache.kafka.clients.admin.DescribeConsumerGroupsResult; +import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions; +import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult; +import org.apache.kafka.clients.admin.ListConsumerGroupsResult; +import org.apache.kafka.clients.admin.MemberDescription; +import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AccessControlEntryFilter; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclBindingFilter; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourcePatternFilter; +import org.apache.kafka.common.resource.ResourceType; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.apache.zookeeper.data.Stat; @@ -61,6 +82,7 @@ import java.lang.management.MemoryMXBean; import java.util.*; import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; /** @@ -479,57 +501,134 @@ public JSONObject zkCliStatus(String clusterAlias) { return target; } - /** - * Create topic to kafka cluster, it is worth noting that the backup number - * must be less than or equal to brokers data. - * - * @param topicName Create topic name. - * @param partitions Create topic partitions. - * @param replic Replic numbers. - * @return Map. - */ - public Map create(String clusterAlias, String topicName, String partitions, String replic) { - Map targets = new HashMap(); - int brokers = getAllBrokersInfo(clusterAlias).size(); - if (Integer.parseInt(replic) > brokers) { - targets.put("status", "error"); - targets.put("info", "replication factor: " + replic + " larger than available brokers: " + brokers); - return targets; - } - Properties prop = new Properties(); - prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); - - if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { - sasl(prop, clusterAlias); - } - if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { - ssl(prop, clusterAlias); - } - - AdminClient adminClient = null; - try { - adminClient = AdminClient.create(prop); - NewTopic newTopic = new NewTopic(topicName, Integer.valueOf(partitions), Short.valueOf(replic)); - adminClient.createTopics(Collections.singleton(newTopic)).all().get(); - } catch (Exception e) { - LOG.info("Create kafka topic has error, msg is " + e.getMessage()); - e.printStackTrace(); - } finally { - adminClient.close(); - } - - targets.put("status", "success"); - targets.put("info", "Create topic[" + topicName + "] has successed,partitions numbers is [" + partitions + "],replication-factor numbers is [" + replic + "]"); - return targets; - } - - /** - * Delete topic to kafka cluster. - */ - public Map delete(String clusterAlias, String topicName) { - Map targets = new HashMap(); - Properties prop = new Properties(); - prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + /** + * Create topic to kafka cluster, it is worth noting that the backup number + * must be less than or equal to brokers data. + * + * @param topicName + * Create topic name. + * @param partitions + * Create topic partitions. + * @param replic + * Replic numbers. + * @return Map. + */ + public Map create(String clusterAlias, String topicName, String partitions, String replic, String username) { + Map targets = new HashMap(); + int brokers = getAllBrokersInfo(clusterAlias).size(); + if (Integer.parseInt(replic) > brokers) { + targets.put("status", "error"); + targets.put("info", "replication factor: " + replic + " larger than available brokers: " + brokers); + return targets; + } + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + NewTopic newTopic = new NewTopic(topicName, Integer.valueOf(partitions), Short.valueOf(replic)); + adminClient.createTopics(Collections.singleton(newTopic)).all().get(); + + + if (!"".equalsIgnoreCase(username)) { + createAclForTopic(adminClient, topicName, username, "group_" + username + "_" + topicName); + } + } catch (Exception e) { + LOG.info("Create kafka topic has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + + targets.put("status", "success"); + targets.put("info", "Create topic[" + topicName + "] has successed,partitions numbers is [" + partitions + "],replication-factor numbers is [" + replic + "]"); + return targets; + } + + private void createAclForTopic(AdminClient client, String topicName, String username, String groupnname) { + String principal = "User:" + username; + String host = "*"; + ArrayList as = new ArrayList<>(); + + ResourcePattern pattern = new ResourcePattern(ResourceType.TOPIC, topicName, PatternType.LITERAL); + + AclBinding aclBindingW = new AclBinding(pattern, new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW)); + AclBinding aclBindingR = new AclBinding(pattern, new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW)); + AclBinding aclBindingD = new AclBinding(pattern, new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW)); + as.add(aclBindingW); + as.add(aclBindingR); + as.add(aclBindingD); + + //group + ResourcePattern resourcePatternGroup = new ResourcePattern(ResourceType.GROUP, groupnname, PatternType.LITERAL); + AclBinding aclBindingG = new AclBinding(resourcePatternGroup, new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW)); + as.add(aclBindingG); + + CreateAclsResult result = client.createAcls(as); + + try { + result.values().get(aclBindingW).get(30, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.info("Error creating acl for topic " + topicName, e); + e.printStackTrace(); + } + } + + + @Override + public Map createGroup(String clusterAlias, String userName, String groupName) { + Map targets = new HashMap(); + + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + String principal = "User:" + userName; + String host = "*"; + ArrayList as = new ArrayList<>(); + //group + ResourcePattern resourcePatternGroup = new ResourcePattern(ResourceType.GROUP, groupName, PatternType.LITERAL); + AclBinding aclBindingG = new AclBinding(resourcePatternGroup, new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW)); + as.add(aclBindingG); + + adminClient.createAcls(as); + } catch (Exception e) { + LOG.info("Create kafka topic has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + + targets.put("status", "success"); + targets.put("info", "Create Group[" + groupName + "] has successed" ); + return targets; + } + + + + + /** Delete topic to kafka cluster. */ + public Map delete(String clusterAlias, String topicName) { + Map targets = new HashMap(); + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { sasl(prop, clusterAlias); @@ -1732,5 +1831,161 @@ public String getUsedMemory(String clusterAlias, String host, int port) { } return memory; } + + + /** + * Get kafka 0.10.x, 1.x, 2.x acl metadata. + */ + public JSONArray getKafkaAcl(String clusterAlias) { + Properties prop = new Properties(); + JSONArray acls = new JSONArray(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + + AclBindingFilter aclBindingFilter = new AclBindingFilter( + ResourcePatternFilter.ANY, + new AccessControlEntryFilter( + null, + null, + AclOperation.ANY, + AclPermissionType.ANY + ) + ); + acls = _getACL(adminClient, aclBindingFilter); + } catch (Exception e) { + LOG.error("Get kafka acl has error,msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + + return acls; + } + + + /** + * Get kafka 0.10.x, 1.x, 2.x acl metadata. + */ + public JSONArray getKafkaAclBYTopicName(String clusterAlias, String topicname) { + Properties prop = new Properties(); + JSONArray acls = new JSONArray(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + + AclBindingFilter aclBindingFilter = new AclBindingFilter( + new ResourcePatternFilter( + ResourceType.TOPIC, + topicname, + PatternType.ANY + ), + AccessControlEntryFilter.ANY + ); + acls = _getACL(adminClient, aclBindingFilter); + } catch (Exception e) { + LOG.error("Get kafka acl has error,msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + + return acls; + } + + + public JSONArray _getACL(AdminClient adminClient, AclBindingFilter aclBindingFilter) throws InterruptedException, ExecutionException { + JSONArray acls = new JSONArray(); + java.util.Iterator itorA = adminClient.describeAcls(aclBindingFilter).values().get().iterator(); + while (itorA.hasNext()) { + AclBinding ab = itorA.next(); + AccessControlEntry e = ab.entry(); + JSONObject entry = new JSONObject(); + entry.put("host", e.host()); + entry.put("operation",e.operation().name()); + entry.put("principal", e.principal()); + entry.put("permissionType",e.permissionType().name()); + + ResourcePattern p = ab.pattern(); + JSONObject pattern = new JSONObject(); + pattern.put("name", p.name()); + pattern.put("patternType", p.patternType().name()); + pattern.put("resourceType", p.resourceType().name()); + + JSONObject acl = new JSONObject(); + acl.put("entry", entry); + acl.put("pattern", pattern); + acls.add(acl); + } + return acls; + } + + @Override + public String deleteAcl(String clusterAlias, JSONObject jsonObject) { + Properties prop = new Properties(); + + String reString = ""; + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + + ArrayList filters = new ArrayList(); + + JSONObject pattern = jsonObject.getJSONObject("pattern"); + ResourcePatternFilter rpFilter = new ResourcePatternFilter( + ResourceType.valueOf(pattern.getString("resourceType")), + pattern.getString("name"), + PatternType.valueOf(pattern.getString("patternType")) + ); + + JSONObject entry = jsonObject.getJSONObject("entry"); + AccessControlEntryFilter arFilter = new AccessControlEntryFilter( + entry.getString("principal"), + entry.getString("host"), + AclOperation.valueOf(entry.getString("operation")), + AclPermissionType.valueOf(entry.getString("permissionType")) + ); + + AclBindingFilter aclBindingFilter = new AclBindingFilter(rpFilter, arFilter); + filters.add(aclBindingFilter); + DeleteAclsResult result = adminClient.deleteAcls(filters); + KafkaFuture kf = result.values().get(aclBindingFilter); + reString = kf.toString();; + } catch (Exception e) { + LOG.error("Get kafka acl has error,msg is " + e.getMessage()); + reString = e.getMessage(); + } finally { + adminClient.close(); + } + + return reString; + } } diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java.orig b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java.orig new file mode 100644 index 00000000..40bd8d9e --- /dev/null +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/KafkaServiceImpl.java.orig @@ -0,0 +1,3180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.core.factory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import kafka.zk.KafkaZkClient; +import org.apache.kafka.clients.CommonClientConfigs; +<<<<<<< HEAD +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ConsumerGroupListing; +import org.apache.kafka.clients.admin.CreateAclsResult; +import org.apache.kafka.clients.admin.DescribeConsumerGroupsResult; +import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions; +import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult; +import org.apache.kafka.clients.admin.ListConsumerGroupsResult; +import org.apache.kafka.clients.admin.MemberDescription; +import org.apache.kafka.clients.admin.NewTopic; +======= +import org.apache.kafka.clients.admin.*; +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.acl.AclOperation; +import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.resource.ResourceType; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.smartloli.kafka.eagle.common.constant.JmxConstants.BrokerServer; +import org.smartloli.kafka.eagle.common.constant.JmxConstants.KafkaServer8; +import org.smartloli.kafka.eagle.common.constant.KSqlParser; +import org.smartloli.kafka.eagle.common.protocol.*; +import org.smartloli.kafka.eagle.common.protocol.topic.TopicPartitionSchema; +import org.smartloli.kafka.eagle.common.util.*; +import org.smartloli.kafka.eagle.common.util.KConstants.BrokerSever; +import org.smartloli.kafka.eagle.common.util.KConstants.CollectorType; +import org.smartloli.kafka.eagle.common.util.KConstants.Kafka; +import scala.Option; +import scala.Tuple2; +import scala.collection.JavaConversions; +import scala.collection.Seq; + +import javax.management.MBeanServerConnection; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXServiceURL; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +/** + * Implements KafkaService all method. + * + * @author smartloli. + *

+ * Created by Jan 18, 2017. + *

+ * Update by hexiang 20170216 + * @see org.smartloli.kafka.eagle.core.factory.KafkaService + */ +public class KafkaServiceImpl implements KafkaService { + +<<<<<<< HEAD + private final String BROKER_IDS_PATH = "/brokers/ids"; + private final String BROKER_TOPICS_PATH = "/brokers/topics"; + private final String CONSUMERS_PATH = "/consumers"; + private final String OWNERS = "/owners"; + private final String TOPIC_ISR = "/brokers/topics/%s/partitions/%s/state"; + private final Logger LOG = LoggerFactory.getLogger(KafkaServiceImpl.class); + + /** Instance Kafka Zookeeper client pool. */ + private KafkaZKPoolUtils kafkaZKPool = KafkaZKPoolUtils.getInstance(); + + /** Zookeeper service interface. */ + private ZkService zkService = new ZkFactory().create(); + + /** + * Find topic and group exist in zookeeper. + * + * @param topic + * Filter topic. + * @param group + * Filter group + * @return Boolean. + */ + public boolean findTopicAndGroupExist(String clusterAlias, String topic, String group) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + String ownersPath = CONSUMERS_PATH + "/" + group + "/owners/" + topic; + boolean status = zkc.pathExists(ownersPath); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return status; + } + + /** + * Obtaining metadata in zookeeper by topic. + * + * @param topic + * Selected condition. + * @return List. + */ + public List findTopicPartition(String clusterAlias, String topic) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Seq brokerTopicsPaths = zkc.getChildren(BROKER_TOPICS_PATH + "/" + topic + "/partitions"); + List topicAndPartitions = JavaConversions.seqAsJavaList(brokerTopicsPaths); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + brokerTopicsPaths = null; + } + return topicAndPartitions; + } + + /** Get kafka active consumer topic. */ + public Map> getActiveTopic(String clusterAlias) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Map> actvTopics = new HashMap>(); + try { + Seq subConsumerPaths = zkc.getChildren(CONSUMERS_PATH); + List groups = JavaConversions.seqAsJavaList(subConsumerPaths); + JSONArray groupsAndTopics = new JSONArray(); + for (String group : groups) { + Seq topics = zkc.getChildren(CONSUMERS_PATH + "/" + group + OWNERS); + for (String topic : JavaConversions.seqAsJavaList(topics)) { + Seq partitionIds = zkc.getChildren(CONSUMERS_PATH + "/" + group + OWNERS + "/" + topic); + if (JavaConversions.seqAsJavaList(partitionIds).size() > 0) { + JSONObject groupAndTopic = new JSONObject(); + groupAndTopic.put("topic", topic); + groupAndTopic.put("group", group); + groupsAndTopics.add(groupAndTopic); + } + } + } + for (Object object : groupsAndTopics) { + JSONObject groupAndTopic = (JSONObject) object; + String group = groupAndTopic.getString("group"); + String topic = groupAndTopic.getString("topic"); + if (actvTopics.containsKey(group + "_" + topic)) { + actvTopics.get(group + "_" + topic).add(topic); + } else { + List topics = new ArrayList(); + topics.add(topic); + actvTopics.put(group + "_" + topic, topics); + } + } + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return actvTopics; + } + + /** Get kafka active consumer topic. */ + public Set getActiveTopic(String clusterAlias, String group) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Set activeTopics = new HashSet<>(); + try { + Seq topics = zkc.getChildren(CONSUMERS_PATH + "/" + group + OWNERS); + for (String topic : JavaConversions.seqAsJavaList(topics)) { + activeTopics.add(topic); + } + } catch (Exception ex) { + LOG.error("Get kafka active topic has error, msg is " + ex.getMessage()); + LOG.error(ex.getMessage()); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return activeTopics; + } + + /** Get all broker list from zookeeper. */ + public List getAllBrokersInfo(String clusterAlias) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + List targets = new ArrayList(); + if (zkc.pathExists(BROKER_IDS_PATH)) { + Seq subBrokerIdsPaths = zkc.getChildren(BROKER_IDS_PATH); + List brokerIdss = JavaConversions.seqAsJavaList(subBrokerIdsPaths); + int id = 0; + for (String ids : brokerIdss) { + try { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_IDS_PATH + "/" + ids); + BrokersInfo broker = new BrokersInfo(); + broker.setCreated(CalendarUtils.convertUnixTime2Date(tuple._2.getCtime())); + broker.setModify(CalendarUtils.convertUnixTime2Date(tuple._2.getMtime())); + String tupleString = new String(tuple._1.get()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable") || SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + String endpoints = JSON.parseObject(tupleString).getString("endpoints"); + String tmp = endpoints.split("//")[1]; + broker.setHost(tmp.substring(0, tmp.length() - 2).split(":")[0]); + broker.setPort(Integer.valueOf(tmp.substring(0, tmp.length() - 2).split(":")[1])); + } else { + String host = JSON.parseObject(tupleString).getString("host"); + int port = JSON.parseObject(tupleString).getInteger("port"); + broker.setHost(host); + broker.setPort(port); + } + broker.setJmxPort(JSON.parseObject(tupleString).getInteger("jmx_port")); + broker.setId(++id); + broker.setIds(ids); + targets.add(broker); + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } + } + } + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return targets; + } + + /** Obtaining kafka consumer information from zookeeper. */ + public Map> getConsumers(String clusterAlias) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Map> consumers = new HashMap>(); + try { + Seq subConsumerPaths = zkc.getChildren(CONSUMERS_PATH); + List groups = JavaConversions.seqAsJavaList(subConsumerPaths); + for (String group : groups) { + String path = CONSUMERS_PATH + "/" + group + "/owners"; + if (zkc.pathExists(path)) { + Seq owners = zkc.getChildren(path); + List ownersSerialize = JavaConversions.seqAsJavaList(owners); + consumers.put(group, ownersSerialize); + } else { + LOG.error("Consumer Path[" + path + "] is not exist."); + } + } + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return consumers; + } + + /** Obtaining kafka consumer page information from zookeeper. */ + public Map> getConsumers(String clusterAlias, DisplayInfo page) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Map> consumers = new HashMap>(); + try { + if (page.getSearch().length() > 0) { + String path = CONSUMERS_PATH + "/" + page.getSearch() + "/owners"; + if (zkc.pathExists(path)) { + Seq owners = zkc.getChildren(path); + List ownersSerialize = JavaConversions.seqAsJavaList(owners); + consumers.put(page.getSearch(), ownersSerialize); + } else { + LOG.error("Consumer Path[" + path + "] is not exist."); + } + } else { + Seq subConsumersPaths = zkc.getChildren(CONSUMERS_PATH); + List groups = JavaConversions.seqAsJavaList(subConsumersPaths); + int offset = 0; + for (String group : groups) { + if (offset < (page.getiDisplayLength() + page.getiDisplayStart()) && offset >= page.getiDisplayStart()) { + String path = CONSUMERS_PATH + "/" + group + "/owners"; + if (zkc.pathExists(path)) { + Seq owners = zkc.getChildren(path); + List ownersSerialize = JavaConversions.seqAsJavaList(owners); + consumers.put(group, ownersSerialize); + } else { + LOG.error("Consumer Path[" + path + "] is not exist."); + } + } + offset++; + } + } + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return consumers; + } + + /** + * According to group, topic and partition to get offset from zookeeper. + * + * @param topic + * Filter topic. + * @param group + * Filter group. + * @param partition + * Filter partition. + * @return OffsetZkInfo. + * + * @see org.smartloli.kafka.eagle.domain.OffsetZkInfo + */ + public OffsetZkInfo getOffset(String clusterAlias, String topic, String group, int partition) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + OffsetZkInfo offsetZk = new OffsetZkInfo(); + String offsetPath = CONSUMERS_PATH + "/" + group + "/offsets/" + topic + "/" + partition; + String ownersPath = CONSUMERS_PATH + "/" + group + "/owners/" + topic + "/" + partition; + Tuple2, Stat> tuple = null; + try { + if (zkc.pathExists(offsetPath)) { + tuple = zkc.getDataAndStat(offsetPath); + } else { + LOG.info("Partition[" + partition + "],OffsetPath[" + offsetPath + "] is not exist!"); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return offsetZk; + } + } catch (Exception ex) { + LOG.error("Partition[" + partition + "],get offset has error,msg is " + ex.getMessage()); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return offsetZk; + } + String tupleString = new String(tuple._1.get()); + long offsetSize = Long.parseLong(tupleString); + if (zkc.pathExists(ownersPath)) { + Tuple2, Stat> tuple2 = zkc.getDataAndStat(ownersPath); + String tupleString2 = new String(tuple2._1.get()); + offsetZk.setOwners(tupleString2 == null ? "" : tupleString2); + } else { + offsetZk.setOwners(""); + } + offsetZk.setOffset(offsetSize); + offsetZk.setCreate(CalendarUtils.convertUnixTime2Date(tuple._2.getCtime())); + offsetZk.setModify(CalendarUtils.convertUnixTime2Date(tuple._2.getMtime())); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return offsetZk; + } + + /** According to topic and partition to obtain Replicas & Isr. */ + public String getReplicasIsr(String clusterAlias, String topic, int partitionid) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + TopicPartition tp = new TopicPartition(topic, partitionid); + Seq replis = zkc.getReplicasForPartition(tp); + List targets = JavaConversions.seqAsJavaList(replis); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return targets.toString(); + } + + /** Get zookeeper cluster information. */ + public String zkCluster(String clusterAlias) { + String[] zks = SystemConfigUtils.getPropertyArray(clusterAlias + ".zk.list", ","); + JSONArray targets = new JSONArray(); + int id = 1; + for (String zk : zks) { + JSONObject object = new JSONObject(); + object.put("id", id++); + object.put("ip", zk.split(":")[0]); + object.put("port", zk.split(":")[1]); + object.put("mode", zkService.status(zk.split(":")[0], zk.split(":")[1])); + targets.add(object); + } + return targets.toJSONString(); + } + + /** Judge whether the zkcli is active. */ + public JSONObject zkCliStatus(String clusterAlias) { + JSONObject target = new JSONObject(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + if (zkc != null) { + target.put("live", true); + target.put("list", SystemConfigUtils.getProperty(clusterAlias + ".zk.list")); + } else { + target.put("live", false); + target.put("list", SystemConfigUtils.getProperty(clusterAlias + ".zk.list")); + } + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return target; + } + + /** + * Create topic to kafka cluster, it is worth noting that the backup number + * must be less than or equal to brokers data. + * + * @param topicName + * Create topic name. + * @param partitions + * Create topic partitions. + * @param replic + * Replic numbers. + * @return Map. + */ + public Map create(String clusterAlias, String topicName, String partitions, String replic, String username) { + Map targets = new HashMap(); + int brokers = getAllBrokersInfo(clusterAlias).size(); + if (Integer.parseInt(replic) > brokers) { + targets.put("status", "error"); + targets.put("info", "replication factor: " + replic + " larger than available brokers: " + brokers); + return targets; + } + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + NewTopic newTopic = new NewTopic(topicName, Integer.valueOf(partitions), Short.valueOf(replic)); + adminClient.createTopics(Collections.singleton(newTopic)).all().get(); + + if (!"".equalsIgnoreCase(username)) { + createAclForTopic(adminClient, topicName, username, "group_" + username + "_" + topicName); + } + + } catch (Exception e) { + LOG.info("Create kafka topic has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + + targets.put("status", "success"); + targets.put("info", "Create topic[" + topicName + "] has successed,partitions numbers is [" + partitions + "],replication-factor numbers is [" + replic + "]"); + return targets; + } + + + private void createAclForTopic(AdminClient client, String topicName, String username, String groupnname) { + String principal = "User:" + username; + String host = "*"; + ArrayList as = new ArrayList<>(); + + ResourcePattern pattern = new ResourcePattern(ResourceType.TOPIC, topicName, PatternType.LITERAL); + + AclBinding aclBindingW = new AclBinding(pattern, new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW)); + AclBinding aclBindingR = new AclBinding(pattern, new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW)); + AclBinding aclBindingD = new AclBinding(pattern, new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW)); + as.add(aclBindingW); + as.add(aclBindingR); + as.add(aclBindingD); + + //group + ResourcePattern resourcePatternGroup = new ResourcePattern(ResourceType.GROUP, groupnname, PatternType.LITERAL); + AclBinding aclBindingG = new AclBinding(resourcePatternGroup, new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW)); + as.add(aclBindingG); + + CreateAclsResult result = client.createAcls(as); + + try { + result.values().get(aclBindingW).get(30, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.info("Error creating acl for topic " + topicName, e); + e.printStackTrace(); + } + } + + + /** Delete topic to kafka cluster. */ + public Map delete(String clusterAlias, String topicName) { + Map targets = new HashMap(); + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + adminClient.deleteTopics(Collections.singleton(topicName)).all().get(); + targets.put("status", "success"); + } catch (Exception e) { + LOG.info("Delete kafka topic has error, msg is " + e.getMessage()); + e.printStackTrace(); + targets.put("status", "failed"); + } finally { + adminClient.close(); + } + return targets; + } + + /** Get kafka brokers from zookeeper. */ + private List getBrokers(String clusterAlias) { + List targets = new ArrayList(); + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + HostsInfo host = new HostsInfo(); + host.setHost(broker.getHost()); + host.setPort(broker.getPort()); + targets.add(host); + } + return targets; + } + + private String parseBrokerServer(String clusterAlias) { + String brokerServer = ""; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + brokerServer += broker.getHost() + ":" + broker.getPort() + ","; + } + if ("".equals(brokerServer)) { + return ""; + } + return brokerServer.substring(0, brokerServer.length() - 1); + } + + /** Convert query sql to object. */ + public KafkaSqlInfo parseSql(String clusterAlias, String sql) { + return segments(clusterAlias, prepare(sql)); + } + + private String prepare(String sql) { + sql = sql.trim(); + sql = sql.replaceAll("\\s+", " "); + return sql; + } + + /** Set topic sasl. */ + public void sasl(Properties props, String clusterAlias) { + // configure the following four settings for SSL Encryption + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.protocol")); + if (!"".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.client.id"))) { + props.put(CommonClientConfigs.CLIENT_ID_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.client.id")); + } + props.put(SaslConfigs.SASL_MECHANISM, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.mechanism")); + props.put(SaslConfigs.SASL_JAAS_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.jaas.config")); + } + + /** Set topic ssl. */ + public void ssl(Properties props, String clusterAlias) { + // configure the following three settings for SSL Encryption + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.protocol")); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.truststore.location")); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.truststore.password")); + + // configure the following three settings for SSL Authentication + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.keystore.location")); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.keystore.password")); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.key.password")); + + } + + private KafkaSqlInfo segments(String clusterAlias, String sql) { + KafkaSqlInfo kafkaSql = new KafkaSqlInfo(); + kafkaSql.setSql(sql); + kafkaSql.getSchema().put("partition", "integer"); + kafkaSql.getSchema().put("offset", "bigint"); + kafkaSql.getSchema().put("msg", "varchar"); + if (!sql.startsWith("select") && !sql.startsWith("SELECT")) { + kafkaSql.setStatus(false); + return kafkaSql; + } else { + String tableName = OdpsSqlParser.parserTopic(sql); + if (!"".equals(tableName)) { + kafkaSql.setStatus(true); + kafkaSql.setTableName(tableName); + } + sql = sql.toLowerCase(); + if (sql.contains("and")) { + sql = sql.split("and")[0]; + } else if (sql.contains("group by")) { + sql = sql.split("group")[0]; + } else if (sql.contains("limit")) { + sql = sql.split("limit")[0]; + } + + Matcher matcher = Pattern.compile("select\\s.+from\\s(.+)where\\s(.+)").matcher(sql); + if (matcher.find()) { + if (matcher.group(2).trim().startsWith("\"partition\"")) { + String[] columns = matcher.group(2).trim().split("in")[1].replace("(", "").replace(")", "").trim().split(","); + for (String column : columns) { + try { + kafkaSql.getPartition().add(Integer.parseInt(column)); + } catch (Exception e) { + LOG.error("Parse parition[" + column + "] has error,msg is " + e.getMessage()); + } + } + } + kafkaSql.setSeeds(getBrokers(clusterAlias)); + } + } + return kafkaSql; + } + + /** Get kafka 0.10.x after activer topics. */ + public Set getKafkaActiverTopics(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + if (!"".equals(consumerGroup.getString("owner")) && consumerGroup.getString("owner") != null) { + topics.add(topic.getString("topic")); + } + } + } + return topics; + } + + public Set getKafkaConsumerTopics(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + return topics; + } + + /** Get kafka 0.10.x, 1.x, 2.x consumer metadata. */ + public String getKafkaConsumer(String clusterAlias) { + Properties prop = new Properties(); + JSONArray consumerGroups = new JSONArray(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult cgrs = adminClient.listConsumerGroups(); + java.util.Iterator itor = cgrs.all().get().iterator(); + while (itor.hasNext()) { + ConsumerGroupListing gs = itor.next(); + JSONObject consumerGroup = new JSONObject(); + String groupId = gs.groupId(); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(groupId)); + if (!groupId.contains("kafka.eagle")) { + consumerGroup.put("group", groupId); + try { + Node node = descConsumerGroup.all().get().get(groupId).coordinator(); + consumerGroup.put("node", node.host() + ":" + node.port()); + } catch (Exception e) { + LOG.error("Get coordinator node has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + consumerGroup.put("meta", getKafkaMetadata(parseBrokerServer(clusterAlias), groupId, clusterAlias)); + consumerGroups.add(consumerGroup); + } + } + } catch (Exception e) { + LOG.error("Get kafka consumer has error,msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return consumerGroups.toJSONString(); + } + + /** Get kafka 0.10.x consumer group & topic information used for page. */ + public String getKafkaConsumer(String clusterAlias, DisplayInfo page) { + Properties prop = new Properties(); + JSONArray consumerGroups = new JSONArray(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult cgrs = adminClient.listConsumerGroups(); + java.util.Iterator itor = cgrs.all().get().iterator(); + if (page.getSearch().length() > 0) { + int offset = 0; + boolean flag = itor.hasNext(); + while (flag) { + ConsumerGroupListing gs = itor.next(); + JSONObject consumerGroup = new JSONObject(); + String groupId = gs.groupId(); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(groupId)); + if (!groupId.contains("kafka.eagle") && groupId.contains(page.getSearch())) { + if (offset < (page.getiDisplayLength() + page.getiDisplayStart()) && offset >= page.getiDisplayStart()) { + consumerGroup.put("group", groupId); + try { + Node node = descConsumerGroup.all().get().get(groupId).coordinator(); + consumerGroup.put("node", node.host() + ":" + node.port()); + } catch (Exception e) { + LOG.error("Get coordinator node has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + consumerGroup.put("meta", getKafkaMetadata(parseBrokerServer(clusterAlias), groupId, clusterAlias)); + consumerGroups.add(consumerGroup); + } + offset++; + } + flag = itor.hasNext(); + if (offset >= page.getiDisplayLength() + page.getiDisplayStart()) { + flag = false; + } + } + } else { + int offset = 0; + boolean flag = itor.hasNext(); + while (flag) { + ConsumerGroupListing gs = itor.next(); + JSONObject consumerGroup = new JSONObject(); + String groupId = gs.groupId(); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(groupId)); + if (!groupId.contains("kafka.eagle")) { + if (offset < (page.getiDisplayLength() + page.getiDisplayStart()) && offset >= page.getiDisplayStart()) { + consumerGroup.put("group", groupId); + try { + Node node = descConsumerGroup.all().get().get(groupId).coordinator(); + consumerGroup.put("node", node.host() + ":" + node.port()); + } catch (Exception e) { + LOG.error("Get coordinator node has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + consumerGroup.put("meta", getKafkaMetadata(parseBrokerServer(clusterAlias), groupId, clusterAlias)); + consumerGroups.add(consumerGroup); + } + offset++; + } + flag = itor.hasNext(); + if (offset >= page.getiDisplayLength() + page.getiDisplayStart()) { + flag = false; + } + } + } + } catch (Exception e) { + LOG.error("Get kafka consumer has error,msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return consumerGroups.toJSONString(); + } + + /** Get kafka 0.10.x consumer metadata. */ + private JSONArray getKafkaMetadata(String bootstrapServers, String group, String clusterAlias) { + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + JSONArray consumerGroups = new JSONArray(); + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(group)); + Collection consumerMetaInfos = descConsumerGroup.describedGroups().get(group).get().members(); + Set hasOwnerTopics = new HashSet<>(); + if (consumerMetaInfos.size() > 0) { + for (MemberDescription consumerMetaInfo : consumerMetaInfos) { + JSONObject topicSub = new JSONObject(); + JSONArray topicSubs = new JSONArray(); + for (TopicPartition topic : consumerMetaInfo.assignment().topicPartitions()) { + JSONObject object = new JSONObject(); + object.put("topic", topic.topic()); + object.put("partition", topic.partition()); + topicSubs.add(object); + hasOwnerTopics.add(topic.topic()); + } + topicSub.put("owner", consumerMetaInfo.consumerId()); + topicSub.put("node", consumerMetaInfo.host().replaceAll("/", "")); + topicSub.put("topicSub", topicSubs); + consumerGroups.add(topicSub); + } + } + + ListConsumerGroupOffsetsResult noActiveTopic = adminClient.listConsumerGroupOffsets(group); + JSONObject topicSub = new JSONObject(); + JSONArray topicSubs = new JSONArray(); + for (Entry entry : noActiveTopic.partitionsToOffsetAndMetadata().get().entrySet()) { + JSONObject object = new JSONObject(); + object.put("topic", entry.getKey().topic()); + object.put("partition", entry.getKey().partition()); + if (!hasOwnerTopics.contains(entry.getKey().topic())) { + topicSubs.add(object); + } + } + topicSub.put("owner", ""); + topicSub.put("node", "-"); + topicSub.put("topicSub", topicSubs); + consumerGroups.add(topicSub); + } catch (Exception e) { + LOG.error("Get kafka consumer metadata has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return consumerGroups; + } + + /** Get kafka 0.10.x consumer pages. */ + public String getKafkaActiverSize(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + int activerCounter = 0; + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + if (!"".equals(consumerGroup.getString("owner")) && consumerGroup.getString("owner") != null) { + activerCounter++; + } + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + JSONObject activerAndTopics = new JSONObject(); + activerAndTopics.put("activers", activerCounter); + activerAndTopics.put("topics", topics.size()); + return activerAndTopics.toJSONString(); + } + + /** Get kafka consumer information pages not owners. */ + public OwnerInfo getKafkaActiverNotOwners(String clusterAlias, String group) { + OwnerInfo ownerInfo = new OwnerInfo(); + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + int activerCounter = 0; + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + if (!"".equals(consumerGroup.getString("owner")) && consumerGroup.getString("owner") != null) { + activerCounter++; + } + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + ownerInfo.setActiveSize(activerCounter); + ownerInfo.setTopicSets(topics); + return ownerInfo; + } + + /** Get kafka 0.10.x, 1.x, 2.x consumer groups. */ + public int getKafkaConsumerGroups(String clusterAlias) { + Properties prop = new Properties(); + int counter = 0; + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups(); + java.util.Iterator groups = consumerGroups.all().get().iterator(); + while (groups.hasNext()) { + String groupId = groups.next().groupId(); + if (!groupId.contains("kafka.eagle")) { + counter++; + } + } + } catch (Exception e) { + LOG.info("Get kafka consumer group has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return counter; + } + + /** Get kafka 0.10.x, 1.x, 2.x consumer topic information. */ + public Set getKafkaConsumerTopic(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + return topics; + } + + /** Get kafka 0.10.x consumer group and topic. */ + public String getKafkaConsumerGroupTopic(String clusterAlias, String group) { + return getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias).toJSONString(); + } + + /** Get kafka 0.10.x, 1.x, 2.x offset from topic. */ + public String getKafkaOffset(String clusterAlias) { + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + JSONArray targets = new JSONArray(); + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups(); + java.util.Iterator groups = consumerGroups.all().get().iterator(); + while (groups.hasNext()) { + String groupId = groups.next().groupId(); + if (!groupId.contains("kafka.eagle")) { + ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(groupId); + for (Entry entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) { + JSONObject object = new JSONObject(); + object.put("group", groupId); + object.put("topic", entry.getKey().topic()); + object.put("partition", entry.getKey().partition()); + object.put("offset", entry.getValue().offset()); + object.put("timestamp", CalendarUtils.getDate()); + targets.add(object); + } + } + } + } catch (Exception e) { + LOG.error("Get consumer offset has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return targets.toJSONString(); + } + + /** Get the data for the topic partition in the specified consumer group */ + public Map getKafkaOffset(String clusterAlias, String group, String topic, Set partitionids) { + Map partitionOffset = new HashMap<>(); + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + List tps = new ArrayList<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + + ListConsumerGroupOffsetsOptions consumerOffsetOptions = new ListConsumerGroupOffsetsOptions(); + consumerOffsetOptions.topicPartitions(tps); + + ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group, consumerOffsetOptions); + for (Entry entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) { + if (topic.equals(entry.getKey().topic())) { + partitionOffset.put(entry.getKey().partition(), entry.getValue().offset()); + } + } + } catch (Exception e) { + LOG.error("Get consumer offset has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return partitionOffset; + } + + /** Get kafka 0.10.x broker bootstrap server. */ + public String getKafkaBrokerServer(String clusterAlias) { + return parseBrokerServer(clusterAlias); + } + + /** Get kafka 0.10.x topic history logsize. */ + public long getKafkaLogSize(String clusterAlias, String topic, int partitionid) { + long histyLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + TopicPartition tp = new TopicPartition(topic, partitionid); + consumer.assign(Collections.singleton(tp)); + java.util.Map logsize = consumer.endOffsets(Collections.singleton(tp)); + try { + histyLogSize = logsize.get(tp).longValue(); + } catch (Exception e) { + LOG.error("Get history topic logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return histyLogSize; + } + + /** Get kafka 0.10.x topic history logsize. */ + public Map getKafkaLogSize(String clusterAlias, String topic, Set partitionids) { + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + Set tps = new HashSet<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + + consumer.assign(tps); + java.util.Map endLogSize = consumer.endOffsets(tps); + if (consumer != null) { + consumer.close(); + } + return endLogSize; + } + + /** Get kafka 0.10.x topic real logsize by partitionid. */ + public long getKafkaRealLogSize(String clusterAlias, String topic, int partitionid) { + long realLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + TopicPartition tp = new TopicPartition(topic, partitionid); + consumer.assign(Collections.singleton(tp)); + java.util.Map endLogSize = consumer.endOffsets(Collections.singleton(tp)); + java.util.Map startLogSize = consumer.beginningOffsets(Collections.singleton(tp)); + try { + realLogSize = endLogSize.get(tp).longValue() - startLogSize.get(tp).longValue(); + } catch (Exception e) { + LOG.error("Get real topic logsize by partition list has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return realLogSize; + } + + /** Get kafka 0.10.x topic real logsize by partitionid set. */ + public long getKafkaRealLogSize(String clusterAlias, String topic, Set partitionids) { + long realLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + Set tps = new HashSet<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + consumer.assign(tps); + java.util.Map endLogSize = consumer.endOffsets(tps); + java.util.Map startLogSize = consumer.beginningOffsets(tps); + try { + long endSumLogSize = 0L; + long startSumLogSize = 0L; + for (Entry entry : endLogSize.entrySet()) { + endSumLogSize += entry.getValue(); + } + for (Entry entry : startLogSize.entrySet()) { + startSumLogSize += entry.getValue(); + } + realLogSize = endSumLogSize - startSumLogSize; + } catch (Exception e) { + LOG.error("Get real topic logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return realLogSize; + } + + /** Get topic producer send logsize records. */ + public long getKafkaProducerLogSize(String clusterAlias, String topic, Set partitionids) { + long producerLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + Set tps = new HashSet<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + consumer.assign(tps); + java.util.Map endLogSize = consumer.endOffsets(tps); + try { + for (Entry entry : endLogSize.entrySet()) { + producerLogSize += entry.getValue(); + } + } catch (Exception e) { + LOG.error("Get producer topic logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return producerLogSize; + } + + /** Get kafka version. */ + public String getKafkaVersion(String host, int port, String ids, String clusterAlias) { + JMXConnector connector = null; + String version = "-"; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + if (CollectorType.KAFKA.equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.offset.storage"))) { + version = mbeanConnection.getAttribute(new ObjectName(String.format(BrokerServer.BROKER_VERSION.getValue(), ids)), BrokerServer.BROKER_VERSION_VALUE.getValue()).toString(); + } else { + version = mbeanConnection.getAttribute(new ObjectName(KafkaServer8.VERSION.getValue()), KafkaServer8.VALUE.getValue()).toString(); + } + } catch (Exception ex) { + LOG.error("Get kafka version from jmx has error, msg is " + ex.getMessage()); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + } + } + } + return version; + } + + /** Get kafka 0.10.x sasl topic metadata. */ + public List findKafkaLeader(String clusterAlias, String topic) { + List targets = new ArrayList<>(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + for (String partition : partitionObject.keySet()) { + String path = String.format(TOPIC_ISR, topic, Integer.valueOf(partition)); + Tuple2, Stat> tuple2 = zkc.getDataAndStat(path); + String tupleString2 = new String(tuple2._1.get()); + JSONObject topicMetadata = JSON.parseObject(tupleString2); + MetadataInfo metadate = new MetadataInfo(); + try { + metadate.setLeader(topicMetadata.getInteger("leader")); + } catch (Exception e) { + LOG.error("Parse string brokerid to int has error, brokerid[" + topicMetadata.getString("leader") + "]"); + e.printStackTrace(); + } + metadate.setPartitionId(Integer.valueOf(partition)); + targets.add(metadate); + } + } + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return targets; + } + + /** Send mock message to kafka topic . */ + public boolean mockMessage(String clusterAlias, String topic, String message) { + Properties props = new Properties(); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(Kafka.KEY_SERIALIZER, StringSerializer.class.getCanonicalName()); + props.put(Kafka.VALUE_SERIALIZER, StringSerializer.class.getCanonicalName()); + props.put(Kafka.PARTITION_CLASS, KafkaPartitioner.class.getName()); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + Producer producer = new KafkaProducer<>(props); + producer.send(new ProducerRecord(topic, new Date().getTime() + "", message)); + producer.close(); + + return true; + } + + /** Get group consumer all topics lags. */ + public long getLag(String clusterAlias, String group, String topic) { + long lag = 0L; + try { + List partitions = findTopicPartition(clusterAlias, topic); + for (String partition : partitions) { + int partitionInt = Integer.parseInt(partition); + OffsetZkInfo offsetZk = getOffset(clusterAlias, topic, group, partitionInt); + long logSize = getLogSize(clusterAlias, topic, partitionInt); + lag += logSize - offsetZk.getOffset(); + } + } catch (Exception e) { + LOG.error("Get cluser[" + clusterAlias + "] active group[" + group + "] topic[" + topic + "] lag has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + return lag; + } + + /** Get kafka group consumer all topics lags. */ + public long getKafkaLag(String clusterAlias, String group, String ketopic) { + long lag = 0L; + + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group); + for (Entry entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) { + if (ketopic.equals(entry.getKey().topic())) { + long logSize = getKafkaLogSize(clusterAlias, entry.getKey().topic(), entry.getKey().partition()); + lag += logSize - entry.getValue().offset(); + } + } + } catch (Exception e) { + LOG.error("Get cluster[" + clusterAlias + "] group[" + group + "] topic[" + ketopic + "] consumer lag has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return lag; + } + + /** Get kafka old version topic history logsize. */ + public long getLogSize(String clusterAlias, String topic, int partitionid) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + logSize = Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** Get kafka old version topic history logsize by partition set. */ + public long getLogSize(String clusterAlias, String topic, Set partitionids) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + for (int partitionid : partitionids) { + logSize += Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + } + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** Get kafka old version real topic logsize. */ + public long getRealLogSize(String clusterAlias, String topic, int partitionid) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + long endLogSize = Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + long startLogSize = Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.START_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + logSize = endLogSize - startLogSize; + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** Get broker host and jmx_port info from ids. */ + public String getBrokerJMXFromIds(String clusterAlias, int ids) { + String jni = ""; + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + if (zkc.pathExists(BROKER_IDS_PATH)) { + try { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_IDS_PATH + "/" + ids); + String tupleString = new String(tuple._1.get()); + String host = ""; + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable") || SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + String endpoints = JSON.parseObject(tupleString).getString("endpoints"); + String tmp = endpoints.split("//")[1]; + host = tmp.substring(0, tmp.length() - 2).split(":")[0]; + } else { + host = JSON.parseObject(tupleString).getString("host"); + } + int jmxPort = JSON.parseObject(tupleString).getInteger("jmx_port"); + jni = host + ":" + jmxPort; + } catch (Exception ex) { + LOG.error("Get broker from ids has error, msg is " + ex.getCause().getMessage()); + ex.printStackTrace(); + } + } + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return jni; + } + + /** Get kafka os memory. */ + public long getOSMemory(String host, int port, String property) { + JMXConnector connector = null; + long memory = 0L; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + MemoryMXBean memBean = ManagementFactory.newPlatformMXBeanProxy(mbeanConnection, ManagementFactory.MEMORY_MXBEAN_NAME, MemoryMXBean.class); + long max = memBean.getHeapMemoryUsage().getMax(); + long used = memBean.getHeapMemoryUsage().getUsed(); + if (BrokerServer.TOTAL_PHYSICAL_MEMORY_SIZE.getValue().equals(property)) { + memory = max; + } else if (BrokerServer.FREE_PHYSICAL_MEMORY_SIZE.getValue().equals(property)) { + memory = max - used; + } + } catch (Exception ex) { + LOG.error("Get kafka os memory from jmx has error, msg is " + ex.getMessage()); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close kafka os memory jmx connector has error, msg is " + e.getMessage()); + } + } + } + return memory; + } +======= + private final String BROKER_IDS_PATH = "/brokers/ids"; + private final String BROKER_TOPICS_PATH = "/brokers/topics"; + private final String CONSUMERS_PATH = "/consumers"; + private final String OWNERS = "/owners"; + private final String TOPIC_ISR = "/brokers/topics/%s/partitions/%s/state"; + private final Logger LOG = LoggerFactory.getLogger(KafkaServiceImpl.class); + + /** + * Instance Kafka Zookeeper client pool. + */ + private KafkaZKPoolUtils kafkaZKPool = KafkaZKPoolUtils.getInstance(); + + /** + * Zookeeper service interface. + */ + private ZkService zkService = new ZkFactory().create(); + + /** + * Find topic and group exist in zookeeper. + * + * @param topic Filter topic. + * @param group Filter group + * @return Boolean. + */ + public boolean findTopicAndGroupExist(String clusterAlias, String topic, String group) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + String ownersPath = CONSUMERS_PATH + "/" + group + "/owners/" + topic; + boolean status = false; + try { + status = zkc.pathExists(ownersPath); + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Find topic and group exist has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return status; + } + + /** + * Obtaining metadata in zookeeper by topic. + * + * @param topic Selected condition. + * @return List. + */ + public List findTopicPartition(String clusterAlias, String topic) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + List topicAndPartitions = null; + Seq brokerTopicsPaths = null; + try { + brokerTopicsPaths = zkc.getChildren(BROKER_TOPICS_PATH + "/" + topic + "/partitions"); + topicAndPartitions = JavaConversions.seqAsJavaList(brokerTopicsPaths); + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Find topic partition has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + brokerTopicsPaths = null; + } + } + return topicAndPartitions; + } + + /** + * Get kafka active consumer topic. + */ + public Map> getActiveTopic(String clusterAlias) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Map> actvTopics = new HashMap>(); + try { + Seq subConsumerPaths = zkc.getChildren(CONSUMERS_PATH); + List groups = JavaConversions.seqAsJavaList(subConsumerPaths); + JSONArray groupsAndTopics = new JSONArray(); + for (String group : groups) { + Seq topics = zkc.getChildren(CONSUMERS_PATH + "/" + group + OWNERS); + for (String topic : JavaConversions.seqAsJavaList(topics)) { + Seq partitionIds = zkc.getChildren(CONSUMERS_PATH + "/" + group + OWNERS + "/" + topic); + if (JavaConversions.seqAsJavaList(partitionIds).size() > 0) { + JSONObject groupAndTopic = new JSONObject(); + groupAndTopic.put("topic", topic); + groupAndTopic.put("group", group); + groupsAndTopics.add(groupAndTopic); + } + } + } + for (Object object : groupsAndTopics) { + JSONObject groupAndTopic = (JSONObject) object; + String group = groupAndTopic.getString("group"); + String topic = groupAndTopic.getString("topic"); + if (actvTopics.containsKey(group + "_" + topic)) { + actvTopics.get(group + "_" + topic).add(topic); + } else { + List topics = new ArrayList(); + topics.add(topic); + actvTopics.put(group + "_" + topic, topics); + } + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get active topic has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return actvTopics; + } + + /** + * Get kafka active consumer topic. + */ + public Set getActiveTopic(String clusterAlias, String group) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Set activeTopics = new HashSet<>(); + try { + Seq topics = zkc.getChildren(CONSUMERS_PATH + "/" + group + OWNERS); + for (String topic : JavaConversions.seqAsJavaList(topics)) { + activeTopics.add(topic); + } + } catch (Exception e) { + LOG.error("Get kafka active topic has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return activeTopics; + } + + /** + * Get all broker list from zookeeper. + */ + public List getAllBrokersInfo(String clusterAlias) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + List targets = new ArrayList(); + try { + if (zkc.pathExists(BROKER_IDS_PATH)) { + Seq subBrokerIdsPaths = zkc.getChildren(BROKER_IDS_PATH); + List brokerIdss = JavaConversions.seqAsJavaList(subBrokerIdsPaths); + int id = 0; + for (String ids : brokerIdss) { + try { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_IDS_PATH + "/" + ids); + BrokersInfo broker = new BrokersInfo(); + broker.setCreated(CalendarUtils.convertUnixTime2Date(tuple._2.getCtime())); + broker.setModify(CalendarUtils.convertUnixTime2Date(tuple._2.getMtime())); + String tupleString = new String(tuple._1.get()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable") || SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + String endpoints = JSON.parseObject(tupleString).getString("endpoints"); + List endpointsList = JSON.parseArray(endpoints, String.class); + String host = ""; + int port = 0; + if (endpointsList.size() > 1) { + String protocol = ""; + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + protocol = Kafka.SASL_PLAINTEXT; + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + protocol = Kafka.SSL; + } + for (String endpointsStr : endpointsList) { + if (endpointsStr.contains(protocol)) { + String tmp = endpointsStr.split("//")[1]; + host = tmp.split(":")[0]; + port = Integer.parseInt(tmp.split(":")[1]); + break; + } + } + } else { + if (endpointsList.size() > 0) { + String tmp = endpointsList.get(0).split("//")[1]; + host = tmp.split(":")[0]; + port = Integer.parseInt(tmp.split(":")[1]); + } + } + broker.setHost(host); + broker.setPort(port); + } else { + String host = JSON.parseObject(tupleString).getString("host"); + int port = JSON.parseObject(tupleString).getInteger("port"); + broker.setHost(host); + broker.setPort(port); + } + broker.setJmxPort(JSON.parseObject(tupleString).getInteger("jmx_port")); + broker.setId(++id); + broker.setIds(ids); + targets.add(broker); + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } + } + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get all brokers info has error,msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return targets; + } + + /** + * Obtaining kafka consumer information from zookeeper. + */ + public Map> getConsumers(String clusterAlias) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Map> consumers = new HashMap>(); + try { + Seq subConsumerPaths = zkc.getChildren(CONSUMERS_PATH); + List groups = JavaConversions.seqAsJavaList(subConsumerPaths); + for (String group : groups) { + String path = CONSUMERS_PATH + "/" + group + "/owners"; + if (zkc.pathExists(path)) { + Seq owners = zkc.getChildren(path); + List ownersSerialize = JavaConversions.seqAsJavaList(owners); + consumers.put(group, ownersSerialize); + } else { + LOG.error("Consumer Path[" + path + "] is not exist."); + } + } + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return consumers; + } + + /** + * Obtaining kafka consumer page information from zookeeper. + */ + public Map> getConsumers(String clusterAlias, DisplayInfo page) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + Map> consumers = new HashMap>(); + try { + if (page.getSearch().length() > 0) { + String path = CONSUMERS_PATH + "/" + page.getSearch() + "/owners"; + if (zkc.pathExists(path)) { + Seq owners = zkc.getChildren(path); + List ownersSerialize = JavaConversions.seqAsJavaList(owners); + consumers.put(page.getSearch(), ownersSerialize); + } else { + LOG.error("Consumer Path[" + path + "] is not exist."); + } + } else { + Seq subConsumersPaths = zkc.getChildren(CONSUMERS_PATH); + List groups = JavaConversions.seqAsJavaList(subConsumersPaths); + int offset = 0; + for (String group : groups) { + if (offset < (page.getiDisplayLength() + page.getiDisplayStart()) && offset >= page.getiDisplayStart()) { + String path = CONSUMERS_PATH + "/" + group + "/owners"; + if (zkc.pathExists(path)) { + Seq owners = zkc.getChildren(path); + List ownersSerialize = JavaConversions.seqAsJavaList(owners); + consumers.put(group, ownersSerialize); + } else { + LOG.error("Consumer Path[" + path + "] is not exist."); + } + } + offset++; + } + } + } catch (Exception ex) { + LOG.error(ex.getMessage()); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return consumers; + } + + /** + * According to group, topic and partition to get offset from zookeeper. + * + * @param topic Filter topic. + * @param group Filter group. + * @param partition Filter partition. + * @return OffsetZkInfo. + * @see org.smartloli.kafka.eagle.common.protocol.OffsetZkInfo + */ + public OffsetZkInfo getOffset(String clusterAlias, String topic, String group, int partition) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + OffsetZkInfo offsetZk = new OffsetZkInfo(); + String offsetPath = CONSUMERS_PATH + "/" + group + "/offsets/" + topic + "/" + partition; + String ownersPath = CONSUMERS_PATH + "/" + group + "/owners/" + topic + "/" + partition; + Tuple2, Stat> tuple = null; + try { + if (zkc.pathExists(offsetPath)) { + tuple = zkc.getDataAndStat(offsetPath); + } else { + LOG.info("Partition[" + partition + "],OffsetPath[" + offsetPath + "] is not exist!"); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return offsetZk; + } + } catch (Exception ex) { + LOG.error("Partition[" + partition + "],get offset has error,msg is " + ex.getMessage()); + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return offsetZk; + } + try { + String tupleString = new String(tuple._1.get()); + long offsetSize = Long.parseLong(tupleString); + if (zkc.pathExists(ownersPath)) { + Tuple2, Stat> tuple2 = zkc.getDataAndStat(ownersPath); + String tupleString2 = new String(tuple2._1.get()); + offsetZk.setOwners(tupleString2 == null ? "" : tupleString2); + } else { + offsetZk.setOwners(""); + } + offsetZk.setOffset(offsetSize); + offsetZk.setCreate(CalendarUtils.convertUnixTime2Date(tuple._2.getCtime())); + offsetZk.setModify(CalendarUtils.convertUnixTime2Date(tuple._2.getMtime())); + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get consumer offsets from zookeeper has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return offsetZk; + } + + /** + * According to topic and partition to obtain Replicas & Isr. + */ + public String getReplicasIsr(String clusterAlias, String topic, int partitionid) { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + TopicPartition tp = new TopicPartition(topic, partitionid); + List targets = null; + try { + Seq replis = zkc.getReplicasForPartition(tp); + targets = JavaConversions.seqAsJavaList(replis); + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get topic replicas isr has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return targets.toString(); + } + + /** + * Get zookeeper cluster information. + */ + public String zkCluster(String clusterAlias) { + String[] zks = SystemConfigUtils.getPropertyArray(clusterAlias + ".zk.list", ","); + JSONArray targets = new JSONArray(); + int id = 1; + for (String zk : zks) { + JSONObject object = new JSONObject(); + object.put("id", id++); + object.put("ip", zk.split(":")[0]); + object.put("port", zk.split(":")[1]); + object.put("mode", zkService.status(zk.split(":")[0], zk.split(":")[1])); + targets.add(object); + } + return targets.toJSONString(); + } + + /** + * Judge whether the zkcli is active. + */ + public JSONObject zkCliStatus(String clusterAlias) { + JSONObject target = new JSONObject(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc != null) { + target.put("live", true); + target.put("list", SystemConfigUtils.getProperty(clusterAlias + ".zk.list")); + } else { + target.put("live", false); + target.put("list", SystemConfigUtils.getProperty(clusterAlias + ".zk.list")); + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get zookeeper client status has error,msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return target; + } + + /** + * Create topic to kafka cluster, it is worth noting that the backup number + * must be less than or equal to brokers data. + * + * @param topicName Create topic name. + * @param partitions Create topic partitions. + * @param replic Replic numbers. + * @return Map. + */ + public Map create(String clusterAlias, String topicName, String partitions, String replic) { + Map targets = new HashMap(); + int brokers = getAllBrokersInfo(clusterAlias).size(); + if (Integer.parseInt(replic) > brokers) { + targets.put("status", "error"); + targets.put("info", "replication factor: " + replic + " larger than available brokers: " + brokers); + return targets; + } + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + NewTopic newTopic = new NewTopic(topicName, Integer.valueOf(partitions), Short.valueOf(replic)); + adminClient.createTopics(Collections.singleton(newTopic)).all().get(); + } catch (Exception e) { + LOG.info("Create kafka topic has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + + targets.put("status", "success"); + targets.put("info", "Create topic[" + topicName + "] has successed,partitions numbers is [" + partitions + "],replication-factor numbers is [" + replic + "]"); + return targets; + } + + /** + * Delete topic to kafka cluster. + */ + public Map delete(String clusterAlias, String topicName) { + Map targets = new HashMap(); + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + adminClient.deleteTopics(Collections.singleton(topicName)).all().get(); + targets.put("status", "success"); + } catch (Exception e) { + LOG.info("Delete kafka topic has error, msg is " + e.getMessage()); + e.printStackTrace(); + targets.put("status", "failed"); + } finally { + adminClient.close(); + } + return targets; + } + + /** + * Get kafka brokers from zookeeper. + */ + private List getBrokers(String clusterAlias) { + List targets = new ArrayList(); + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + HostsInfo host = new HostsInfo(); + host.setHost(broker.getHost()); + host.setPort(broker.getPort()); + targets.add(host); + } + return targets; + } + + private String parseBrokerServer(String clusterAlias) { + String brokerServer = ""; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + brokerServer += broker.getHost() + ":" + broker.getPort() + ","; + } + if ("".equals(brokerServer)) { + return ""; + } + return brokerServer.substring(0, brokerServer.length() - 1); + } + + /** + * Convert query sql to object. + */ + public KafkaSqlInfo parseSql(String clusterAlias, String sql) { + return segments(clusterAlias, prepare(sql)); + } + + private String prepare(String sql) { + sql = sql.trim(); + sql = sql.replaceAll("\\s+", " "); + return sql; + } + + /** + * Set topic sasl. + */ + public void sasl(Properties props, String clusterAlias) { + // configure the following four settings for SSL Encryption + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.protocol")); + if (!"".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.client.id"))) { + props.put(CommonClientConfigs.CLIENT_ID_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.client.id")); + } + props.put(SaslConfigs.SASL_MECHANISM, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.mechanism")); + props.put(SaslConfigs.SASL_JAAS_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.sasl.jaas.config")); + } + + /** + * Set topic ssl. + */ + public void ssl(Properties props, String clusterAlias) { + // configure the following three settings for SSL Encryption + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.protocol")); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.truststore.location")); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.truststore.password")); + + // configure the following three settings for SSL Authentication + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.keystore.location")); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.keystore.password")); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.ssl.key.password")); + + } + + private KafkaSqlInfo segments(String clusterAlias, String sql) { + KafkaSqlInfo kafkaSql = new KafkaSqlInfo(); + kafkaSql.setSql(sql); + kafkaSql.getSchema().put("partition", "integer"); + kafkaSql.getSchema().put("offset", "bigint"); + kafkaSql.getSchema().put("msg", "varchar"); + kafkaSql.getSchema().put("timespan", "varchar"); + kafkaSql.getSchema().put("date", "varchar"); + if (!sql.startsWith("select") && !sql.startsWith("SELECT")) { + kafkaSql.setStatus(false); + return kafkaSql; + } else { + TopicPartitionSchema tps = KSqlParser.parserTopic(sql); + if (tps != null && !"".equals(tps.getTopic())) { + kafkaSql.setStatus(true); + kafkaSql.setTableName(tps.getTopic()); + kafkaSql.setSeeds(getBrokers(clusterAlias)); + kafkaSql.setPartition(tps.getPartitions()); + kafkaSql.setLimit(tps.getLimit()); + } + } + return kafkaSql; + } + + /** + * Get kafka 0.10.x after activer topics. + */ + public Set getKafkaActiverTopics(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + if (!"".equals(consumerGroup.getString("owner")) && consumerGroup.getString("owner") != null) { + topics.add(topic.getString("topic")); + } + } + } + return topics; + } + + public Set getKafkaConsumerTopics(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + return topics; + } + + /** + * Get kafka 0.10.x, 1.x, 2.x consumer metadata. + */ + public String getKafkaConsumer(String clusterAlias) { + Properties prop = new Properties(); + JSONArray consumerGroups = new JSONArray(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult cgrs = adminClient.listConsumerGroups(); + java.util.Iterator itor = cgrs.all().get().iterator(); + while (itor.hasNext()) { + ConsumerGroupListing gs = itor.next(); + JSONObject consumerGroup = new JSONObject(); + String groupId = gs.groupId(); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(groupId)); + if (!groupId.contains("kafka.eagle")) { + consumerGroup.put("group", groupId); + try { + Node node = descConsumerGroup.all().get().get(groupId).coordinator(); + consumerGroup.put("node", node.host() + ":" + node.port()); + } catch (Exception e) { + LOG.error("Get coordinator node has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + consumerGroup.put("meta", getKafkaMetadata(parseBrokerServer(clusterAlias), groupId, clusterAlias)); + consumerGroups.add(consumerGroup); + } + } + } catch (Exception e) { + LOG.error("Get kafka consumer has error,msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return consumerGroups.toJSONString(); + } + + /** + * Get kafka 0.10.x consumer group & topic information used for page. + */ + public String getKafkaConsumer(String clusterAlias, DisplayInfo page) { + Properties prop = new Properties(); + JSONArray consumerGroups = new JSONArray(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult cgrs = adminClient.listConsumerGroups(); + java.util.Iterator itor = cgrs.all().get().iterator(); + if (page.getSearch().length() > 0) { + int offset = 0; + boolean flag = itor.hasNext(); + while (flag) { + ConsumerGroupListing gs = itor.next(); + JSONObject consumerGroup = new JSONObject(); + String groupId = gs.groupId(); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(groupId)); + if (!groupId.contains("kafka.eagle") && groupId.contains(page.getSearch())) { + if (offset < (page.getiDisplayLength() + page.getiDisplayStart()) && offset >= page.getiDisplayStart()) { + consumerGroup.put("group", groupId); + try { + Node node = descConsumerGroup.all().get().get(groupId).coordinator(); + consumerGroup.put("node", node.host() + ":" + node.port()); + } catch (Exception e) { + LOG.error("Get coordinator node has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + consumerGroup.put("meta", getKafkaMetadata(parseBrokerServer(clusterAlias), groupId, clusterAlias)); + consumerGroups.add(consumerGroup); + } + offset++; + } + flag = itor.hasNext(); + if (offset >= page.getiDisplayLength() + page.getiDisplayStart()) { + flag = false; + } + } + } else { + int offset = 0; + boolean flag = itor.hasNext(); + while (flag) { + ConsumerGroupListing gs = itor.next(); + JSONObject consumerGroup = new JSONObject(); + String groupId = gs.groupId(); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(groupId)); + if (!groupId.contains("kafka.eagle")) { + if (offset < (page.getiDisplayLength() + page.getiDisplayStart()) && offset >= page.getiDisplayStart()) { + consumerGroup.put("group", groupId); + try { + Node node = descConsumerGroup.all().get().get(groupId).coordinator(); + consumerGroup.put("node", node.host() + ":" + node.port()); + } catch (Exception e) { + LOG.error("Get coordinator node has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + consumerGroup.put("meta", getKafkaMetadata(parseBrokerServer(clusterAlias), groupId, clusterAlias)); + consumerGroups.add(consumerGroup); + } + offset++; + } + flag = itor.hasNext(); + if (offset >= page.getiDisplayLength() + page.getiDisplayStart()) { + flag = false; + } + } + } + } catch (Exception e) { + LOG.error("Get kafka consumer has error,msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return consumerGroups.toJSONString(); + } + + /** + * Get kafka 0.10.x consumer metadata. + */ + private JSONArray getKafkaMetadata(String bootstrapServers, String group, String clusterAlias) { + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + JSONArray consumerGroups = new JSONArray(); + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + DescribeConsumerGroupsResult descConsumerGroup = adminClient.describeConsumerGroups(Arrays.asList(group)); + Collection consumerMetaInfos = descConsumerGroup.describedGroups().get(group).get().members(); + Set hasOwnerTopics = new HashSet<>(); + if (consumerMetaInfos.size() > 0) { + for (MemberDescription consumerMetaInfo : consumerMetaInfos) { + JSONObject topicSub = new JSONObject(); + JSONArray topicSubs = new JSONArray(); + for (TopicPartition topic : consumerMetaInfo.assignment().topicPartitions()) { + JSONObject object = new JSONObject(); + object.put("topic", topic.topic()); + object.put("partition", topic.partition()); + topicSubs.add(object); + hasOwnerTopics.add(topic.topic()); + } + topicSub.put("owner", consumerMetaInfo.consumerId()); + topicSub.put("node", consumerMetaInfo.host().replaceAll("/", "")); + topicSub.put("topicSub", topicSubs); + consumerGroups.add(topicSub); + } + } + + ListConsumerGroupOffsetsResult noActiveTopic = adminClient.listConsumerGroupOffsets(group); + JSONObject topicSub = new JSONObject(); + JSONArray topicSubs = new JSONArray(); + for (Entry entry : noActiveTopic.partitionsToOffsetAndMetadata().get().entrySet()) { + JSONObject object = new JSONObject(); + object.put("topic", entry.getKey().topic()); + object.put("partition", entry.getKey().partition()); + if (!hasOwnerTopics.contains(entry.getKey().topic())) { + topicSubs.add(object); + } + } + topicSub.put("owner", ""); + topicSub.put("node", "-"); + topicSub.put("topicSub", topicSubs); + consumerGroups.add(topicSub); + } catch (Exception e) { + LOG.error("Get kafka consumer metadata has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return consumerGroups; + } + + /** + * Get kafka 0.10.x consumer pages. + */ + public String getKafkaActiverSize(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + int activerCounter = 0; + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + if (!"".equals(consumerGroup.getString("owner")) && consumerGroup.getString("owner") != null) { + activerCounter++; + } + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + JSONObject activerAndTopics = new JSONObject(); + activerAndTopics.put("activers", activerCounter); + activerAndTopics.put("topics", topics.size()); + return activerAndTopics.toJSONString(); + } + + /** + * Get kafka consumer information pages not owners. + */ + public OwnerInfo getKafkaActiverNotOwners(String clusterAlias, String group) { + OwnerInfo ownerInfo = new OwnerInfo(); + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + int activerCounter = 0; + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + if (!"".equals(consumerGroup.getString("owner")) && consumerGroup.getString("owner") != null) { + activerCounter++; + } + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + ownerInfo.setActiveSize(activerCounter); + ownerInfo.setTopicSets(topics); + return ownerInfo; + } + + /** + * Get kafka 0.10.x, 1.x, 2.x consumer groups. + */ + public int getKafkaConsumerGroups(String clusterAlias) { + Properties prop = new Properties(); + int counter = 0; + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups(); + java.util.Iterator groups = consumerGroups.all().get().iterator(); + while (groups.hasNext()) { + String groupId = groups.next().groupId(); + if (!groupId.contains("kafka.eagle")) { + counter++; + } + } + } catch (Exception e) { + LOG.info("Get kafka consumer group has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return counter; + } + + /** + * Get kafka 0.10.x, 1.x, 2.x consumer topic information. + */ + public Set getKafkaConsumerTopic(String clusterAlias, String group) { + JSONArray consumerGroups = getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias); + Set topics = new HashSet<>(); + for (Object object : consumerGroups) { + JSONObject consumerGroup = (JSONObject) object; + for (Object topicObject : consumerGroup.getJSONArray("topicSub")) { + JSONObject topic = (JSONObject) topicObject; + topics.add(topic.getString("topic")); + } + } + return topics; + } + + /** + * Get kafka 0.10.x consumer group and topic. + */ + public String getKafkaConsumerGroupTopic(String clusterAlias, String group) { + return getKafkaMetadata(parseBrokerServer(clusterAlias), group, clusterAlias).toJSONString(); + } + + /** + * Get kafka 0.10.x, 1.x, 2.x offset from topic. + */ + public String getKafkaOffset(String clusterAlias) { + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + JSONArray targets = new JSONArray(); + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups(); + java.util.Iterator groups = consumerGroups.all().get().iterator(); + while (groups.hasNext()) { + String groupId = groups.next().groupId(); + if (!groupId.contains("kafka.eagle")) { + ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(groupId); + for (Entry entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) { + JSONObject object = new JSONObject(); + object.put("group", groupId); + object.put("topic", entry.getKey().topic()); + object.put("partition", entry.getKey().partition()); + object.put("offset", entry.getValue().offset()); + object.put("timestamp", CalendarUtils.getDate()); + targets.add(object); + } + } + } + } catch (Exception e) { + LOG.error("Get consumer offset has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return targets.toJSONString(); + } + + /** + * Get the data for the topic partition in the specified consumer group + */ + public Map getKafkaOffset(String clusterAlias, String group, String topic, Set partitionids) { + Map partitionOffset = new HashMap<>(); + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + List tps = new ArrayList<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + + ListConsumerGroupOffsetsOptions consumerOffsetOptions = new ListConsumerGroupOffsetsOptions(); + consumerOffsetOptions.topicPartitions(tps); + + ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group, consumerOffsetOptions); + for (Entry entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) { + if (topic.equals(entry.getKey().topic())) { + partitionOffset.put(entry.getKey().partition(), entry.getValue().offset()); + } + } + } catch (Exception e) { + LOG.error("Get consumer offset has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return partitionOffset; + } + + /** + * Get kafka 0.10.x broker bootstrap server. + */ + public String getKafkaBrokerServer(String clusterAlias) { + return parseBrokerServer(clusterAlias); + } + + /** + * Get kafka 0.10.x topic history logsize. + */ + public long getKafkaLogSize(String clusterAlias, String topic, int partitionid) { + long histyLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + TopicPartition tp = new TopicPartition(topic, partitionid); + consumer.assign(Collections.singleton(tp)); + java.util.Map logsize = consumer.endOffsets(Collections.singleton(tp)); + try { + histyLogSize = logsize.get(tp).longValue(); + } catch (Exception e) { + LOG.error("Get history topic logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return histyLogSize; + } + + /** + * Get kafka 0.10.x topic history logsize. + */ + public Map getKafkaLogSize(String clusterAlias, String topic, Set partitionids) { + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + Set tps = new HashSet<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + + consumer.assign(tps); + java.util.Map endLogSize = consumer.endOffsets(tps); + if (consumer != null) { + consumer.close(); + } + return endLogSize; + } + + /** + * Get kafka 0.10.x topic real logsize by partitionid. + */ + public long getKafkaRealLogSize(String clusterAlias, String topic, int partitionid) { + long realLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + TopicPartition tp = new TopicPartition(topic, partitionid); + consumer.assign(Collections.singleton(tp)); + java.util.Map endLogSize = consumer.endOffsets(Collections.singleton(tp)); + java.util.Map startLogSize = consumer.beginningOffsets(Collections.singleton(tp)); + try { + realLogSize = endLogSize.get(tp).longValue() - startLogSize.get(tp).longValue(); + } catch (Exception e) { + LOG.error("Get real topic logsize by partition list has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return realLogSize; + } + + /** + * Get kafka 0.10.x topic real logsize by partitionid set. + */ + public long getKafkaRealLogSize(String clusterAlias, String topic, Set partitionids) { + long realLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + Set tps = new HashSet<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + consumer.assign(tps); + java.util.Map endLogSize = consumer.endOffsets(tps); + java.util.Map startLogSize = consumer.beginningOffsets(tps); + try { + long endSumLogSize = 0L; + long startSumLogSize = 0L; + for (Entry entry : endLogSize.entrySet()) { + endSumLogSize += entry.getValue(); + } + for (Entry entry : startLogSize.entrySet()) { + startSumLogSize += entry.getValue(); + } + realLogSize = endSumLogSize - startSumLogSize; + } catch (Exception e) { + LOG.error("Get real topic logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return realLogSize; + } + + /** + * Get topic producer send logsize records. + */ + public long getKafkaProducerLogSize(String clusterAlias, String topic, Set partitionids) { + long producerLogSize = 0L; + Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName()); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + KafkaConsumer consumer = new KafkaConsumer<>(props); + Set tps = new HashSet<>(); + for (int partitionid : partitionids) { + TopicPartition tp = new TopicPartition(topic, partitionid); + tps.add(tp); + } + consumer.assign(tps); + java.util.Map endLogSize = consumer.endOffsets(tps); + try { + for (Entry entry : endLogSize.entrySet()) { + producerLogSize += entry.getValue(); + } + } catch (Exception e) { + LOG.error("Get producer topic logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + if (consumer != null) { + consumer.close(); + } + } + return producerLogSize; + } + + /** + * Get kafka version. + */ + public String getKafkaVersion(String host, int port, String ids, String clusterAlias) { + JMXConnector connector = null; + String version = "-"; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + if (CollectorType.KAFKA.equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.offset.storage"))) { + version = mbeanConnection.getAttribute(new ObjectName(String.format(BrokerServer.BROKER_VERSION.getValue(), ids)), BrokerServer.BROKER_VERSION_VALUE.getValue()).toString(); + } else { + version = mbeanConnection.getAttribute(new ObjectName(KafkaServer8.VERSION.getValue()), KafkaServer8.VALUE.getValue()).toString(); + } + } catch (Exception ex) { + LOG.error("Get kafka version from jmx has error, msg is " + ex.getMessage()); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + } + } + } + return version; + } + + /** + * Get kafka 0.10.x sasl topic metadata. + */ + public List findKafkaLeader(String clusterAlias, String topic) { + List targets = new ArrayList<>(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + for (String partition : partitionObject.keySet()) { + String path = String.format(TOPIC_ISR, topic, Integer.valueOf(partition)); + Tuple2, Stat> tuple2 = zkc.getDataAndStat(path); + String tupleString2 = new String(tuple2._1.get()); + JSONObject topicMetadata = JSON.parseObject(tupleString2); + MetadataInfo metadate = new MetadataInfo(); + try { + metadate.setLeader(topicMetadata.getInteger("leader")); + } catch (Exception e) { + LOG.error("Parse string brokerid to int has error, brokerid[" + topicMetadata.getString("leader") + "]"); + e.printStackTrace(); + } + metadate.setPartitionId(Integer.valueOf(partition)); + targets.add(metadate); + } + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Find kafka partition leader has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return targets; + } + + /** + * Send mock message to kafka topic . + */ + public boolean mockMessage(String clusterAlias, String topic, String message) { + Properties props = new Properties(); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); + props.put(Kafka.KEY_SERIALIZER, StringSerializer.class.getCanonicalName()); + props.put(Kafka.VALUE_SERIALIZER, StringSerializer.class.getCanonicalName()); + props.put(Kafka.PARTITION_CLASS, KafkaPartitioner.class.getName()); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(props, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(props, clusterAlias); + } + Producer producer = new KafkaProducer<>(props); + producer.send(new ProducerRecord(topic, new Date().getTime() + "", message)); + producer.close(); + + return true; + } + + /** + * Get group consumer all topics lags. + */ + public long getLag(String clusterAlias, String group, String topic) { + long lag = 0L; + try { + List partitions = findTopicPartition(clusterAlias, topic); + for (String partition : partitions) { + int partitionInt = Integer.parseInt(partition); + OffsetZkInfo offsetZk = getOffset(clusterAlias, topic, group, partitionInt); + long logSize = getLogSize(clusterAlias, topic, partitionInt); + lag += logSize - offsetZk.getOffset(); + } + } catch (Exception e) { + LOG.error("Get cluser[" + clusterAlias + "] active group[" + group + "] topic[" + topic + "] lag has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + return lag; + } + + /** + * Get kafka group consumer all topics lags. + */ + public long getKafkaLag(String clusterAlias, String group, String ketopic) { + long lag = 0L; + + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + ssl(prop, clusterAlias); + } + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group); + for (Entry entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) { + if (ketopic.equals(entry.getKey().topic())) { + long logSize = getKafkaLogSize(clusterAlias, entry.getKey().topic(), entry.getKey().partition()); + lag += logSize - entry.getValue().offset(); + } + } + } catch (Exception e) { + LOG.error("Get cluster[" + clusterAlias + "] group[" + group + "] topic[" + ketopic + "] consumer lag has error, msg is " + e.getMessage()); + e.printStackTrace(); + } finally { + adminClient.close(); + } + return lag; + } + + /** + * Get kafka old version topic history logsize. + */ + public long getLogSize(String clusterAlias, String topic, int partitionid) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + logSize = Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** + * Get kafka old version topic history logsize by partition set. + */ + public long getLogSize(String clusterAlias, String topic, Set partitionids) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + for (int partitionid : partitionids) { + logSize += Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + } + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** + * Get kafka old version real topic logsize. + */ + public long getRealLogSize(String clusterAlias, String topic, int partitionid) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + long endLogSize = Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + long startLogSize = Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.START_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + logSize = endLogSize - startLogSize; + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** + * Get kafka old version real topic logsize. + */ + public long getRealLogSize(String clusterAlias, String topic, Set partitionids) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + List brokers = getAllBrokersInfo(clusterAlias); + for (BrokersInfo broker : brokers) { + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, broker.getHost() + ":" + broker.getJmxPort())); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + if (connector != null) { + break; + } + } catch (Exception e) { + LOG.error("Get kafka old version logsize has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + long logSize = 0L; + try { + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + long endLogSize = 0L; + long startLogSize = 0L; + for (int partitionid : partitionids) { + endLogSize += Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.END_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + startLogSize += Long.parseLong(mbeanConnection.getAttribute(new ObjectName(String.format(KafkaServer8.START_LOG_SIZE.getValue(), topic, partitionid)), KafkaServer8.VALUE.getValue()).toString()); + } + logSize = endLogSize - startLogSize; + } catch (Exception ex) { + LOG.error("Get kafka old version logsize & parse has error, msg is " + ex.getMessage()); + ex.printStackTrace(); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close jmx connector has error, msg is " + e.getMessage()); + e.printStackTrace(); + } + } + } + return logSize; + } + + /** + * Get broker host and jmx_port info from ids. + */ + public String getBrokerJMXFromIds(String clusterAlias, int ids) { + String jni = ""; + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_IDS_PATH)) { + try { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_IDS_PATH + "/" + ids); + String tupleString = new String(tuple._1.get()); + String host = ""; + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable") || SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + String endpoints = JSON.parseObject(tupleString).getString("endpoints"); + String tmp = endpoints.split("//")[1]; + host = tmp.substring(0, tmp.length() - 2).split(":")[0]; + } else { + host = JSON.parseObject(tupleString).getString("host"); + } + int jmxPort = JSON.parseObject(tupleString).getInteger("jmx_port"); + jni = host + ":" + jmxPort; + } catch (Exception ex) { + LOG.error("Get broker from ids has error, msg is " + ex.getCause().getMessage()); + ex.printStackTrace(); + } + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get broker jmx info from ids has error,msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return jni; + } + + /** + * Get kafka os memory. + */ + public long getOSMemory(String clusterAlias, String host, int port, String property) { + JMXConnector connector = null; + long memory = 0L; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + MemoryMXBean memBean = ManagementFactory.newPlatformMXBeanProxy(mbeanConnection, ManagementFactory.MEMORY_MXBEAN_NAME, MemoryMXBean.class); + long max = memBean.getHeapMemoryUsage().getMax(); + long used = memBean.getHeapMemoryUsage().getUsed(); + if (BrokerServer.TOTAL_PHYSICAL_MEMORY_SIZE.getValue().equals(property)) { + memory = max; + } else if (BrokerServer.FREE_PHYSICAL_MEMORY_SIZE.getValue().equals(property)) { + memory = max - used; + } + } catch (Exception ex) { + LOG.error("Get kafka os memory from jmx has error, msg is " + ex.getMessage()); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close kafka os memory jmx connector has error, msg is " + e.getMessage()); + } + } + } + return memory; + } + + /** + * Get kafka cpu. + */ + public String getUsedCpu(String clusterAlias, String host, int port) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + String cpu = "NULL"; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + String value = mbeanConnection.getAttribute(new ObjectName(BrokerServer.JMX_PERFORMANCE_TYPE.getValue()), BrokerServer.PROCESS_CPU_LOAD.getValue()).toString(); + double cpuValue = Double.parseDouble(value); + String percent = StrUtils.numberic((cpuValue * 100.0) + "") + "%"; + if ((cpuValue * 100.0) < BrokerSever.CPU_NORMAL) { + cpu = "" + percent + ""; + } else if ((cpuValue * 100.0) >= BrokerSever.CPU_NORMAL && (cpuValue * 100.0) < BrokerSever.CPU_DANGER) { + cpu = "" + percent + ""; + } else if ((cpuValue * 100.0) >= BrokerSever.CPU_DANGER) { + cpu = "" + percent + ""; + } + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Get kafka broker used cpu has error, msg is ", e); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close kafka broker cpu jmx connector has error, msg is " + e.getMessage()); + } + } + } + return cpu; + } + + /** + * Get kafka cpu value. + */ + public double getUsedCpuValue(String clusterAlias, String host, int port) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + double cpu = 0.00; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + String value = mbeanConnection.getAttribute(new ObjectName(BrokerServer.JMX_PERFORMANCE_TYPE.getValue()), BrokerServer.PROCESS_CPU_LOAD.getValue()).toString(); + double cpuValue = Double.parseDouble(value); + cpu = StrUtils.numberic(String.valueOf(cpuValue * 100.0)); + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Get kafka broker used cpu value has error, msg is ", e); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close kafka used cpu value jmx connector has error, msg is " + e.getMessage()); + } + } + } + return cpu; + } + + /** + * Get kafka used memory. + */ + public String getUsedMemory(String clusterAlias, String host, int port) { + JMXConnector connector = null; + String JMX = "service:jmx:rmi:///jndi/rmi://%s/jmxrmi"; + String memory = "NULL"; + try { + JMXServiceURL jmxSeriverUrl = new JMXServiceURL(String.format(JMX, host + ":" + port)); + connector = JMXFactoryUtils.connectWithTimeout(clusterAlias, jmxSeriverUrl, 30, TimeUnit.SECONDS); + MBeanServerConnection mbeanConnection = connector.getMBeanServerConnection(); + MemoryMXBean memBean = ManagementFactory.newPlatformMXBeanProxy(mbeanConnection, ManagementFactory.MEMORY_MXBEAN_NAME, MemoryMXBean.class); + long used = memBean.getHeapMemoryUsage().getUsed(); + long max = memBean.getHeapMemoryUsage().getMax(); + String percent = StrUtils.stringify(used) + " (" + StrUtils.numberic((used * 100.0 / max) + "") + "%)"; + if ((used * 100.0) / max < BrokerSever.MEM_NORMAL) { + memory = "" + percent + ""; + } else if ((used * 100.0) / max >= BrokerSever.MEM_NORMAL && (used * 100.0) / max < BrokerSever.MEM_DANGER) { + memory = "" + percent + ""; + } else if ((used * 100.0) / max >= BrokerSever.MEM_DANGER) { + memory = "" + percent + ""; + } + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Get kafka broker used memroy has error, msg is ", e); + } finally { + if (connector != null) { + try { + connector.close(); + } catch (IOException e) { + LOG.error("Close kafka used memory jmx connector has error, msg is " + e.getMessage()); + } + } + } + return memory; + } +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 + +} diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigFactory.java b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigFactory.java new file mode 100644 index 00000000..a1d2a23e --- /dev/null +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigFactory.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.core.factory.v2; + +/** +* Topic factory implements {@link BrokerProvider} service. +* +* @author jeff. +* +* Created by Jun 13, 2019 +*/ +public class ConfigFactory implements ConfigProvider{ + + @Override + public ConfigService create() { + return new ConfigServiceImpl(); + } + +} diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigProvider.java b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigProvider.java new file mode 100644 index 00000000..07b60f22 --- /dev/null +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigProvider.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.core.factory.v2; + +/** +* Topic provider {@link BrokerService} visitor enter. +* +* @author jeff. +* +*/ +public interface ConfigProvider { + + public ConfigService create(); + +} diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigService.java b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigService.java new file mode 100644 index 00000000..40494779 --- /dev/null +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigService.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.core.factory.v2; + +import java.util.List; +import java.util.Map; + +import org.smartloli.kafka.eagle.common.protocol.MetadataInfo; +import org.smartloli.kafka.eagle.common.protocol.PartitionsInfo; + +/** + * Consumer group, topic and topic page or partition interface.Kafka api 2.x + * version. + * + * @author jeff. + * + */ +public interface ConfigService { + + /** Check topic from zookeeper metadata. */ + public boolean findKafkaTopic(String clusterAlias, String topic); + + /** Get topic list. */ + public List topicList(String clusterAlias); + + /** Get select topic list. */ + public String topicListParams(String clusterAlias, String search); + + /** Get kafka broker numbers. */ + public long brokerNumbers(String clusterAlias); + + /** Get topic number from zookeeper. */ + public long topicNumbers(String clusterAlias); + + /** Get topic number with match name from zookeeper. */ + public long topicNumbers(String clusterAlias, String topic); + + /** Get partition number from zookeeper. */ + public long partitionNumbers(String clusterAlias, String topic); + + /** Scan topic page display. */ + public List topicRecords(String clusterAlias, Map params); + + /** Scan topic meta page display. */ + public List topicMetadataRecords(String clusterAlias, String topic, Map params); + + /** Get topic producer logsize total. */ + public long getTopicLogSizeTotal(String clusterAlias, String topic); + + /** Get topic real logsize records. */ + public long getTopicRealLogSize(String clusterAlias, String topic); + + /** Get topic producer send logsize records. */ + public long getTopicProducerLogSize(String clusterAlias, String topic); + + /** Add topic partitions. */ + public Map createTopicPartitions(String clusterAlias, String topic, int totalCount); + + /** Get broker spread by topic. */ + public int getBrokerSpreadByTopic(String clusterAlias, String topic); + + /** Get broker skewed by topic. */ + public int getBrokerSkewedByTopic(String clusterAlias, String topic); + + /** Get broker leader skewed by topic. */ + public int getBrokerLeaderSkewedByTopic(String clusterAlias, String topic); + + /** Get broker id list. */ + public List getBrokerIdList(String clusterAlias); + + /** + * Get kafka broker numbers from zookeeper. + * @return + */ + long getUserList(String clusterAlias); + +} diff --git a/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigServiceImpl.java b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigServiceImpl.java new file mode 100644 index 00000000..ccb5a5c8 --- /dev/null +++ b/kafka-eagle-core/src/main/java/org/smartloli/kafka/eagle/core/factory/v2/ConfigServiceImpl.java @@ -0,0 +1,764 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.core.factory.v2; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.TypeReference; +import com.google.common.base.Strings; +import kafka.zk.KafkaZkClient; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewPartitions; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.smartloli.kafka.eagle.common.protocol.MetadataInfo; +import org.smartloli.kafka.eagle.common.protocol.PartitionsInfo; +import org.smartloli.kafka.eagle.common.util.*; +import org.smartloli.kafka.eagle.common.util.KConstants.Kafka; +import org.smartloli.kafka.eagle.core.factory.KafkaFactory; +import org.smartloli.kafka.eagle.core.factory.KafkaService; +import scala.Option; +import scala.Tuple2; +import scala.collection.JavaConversions; +import scala.collection.Seq; + +import java.util.*; +import java.util.Map.Entry; + +/** + * Implements {@link ConfigService} all method. + * + * @author jeff. + */ +public class ConfigServiceImpl implements ConfigService { + + private final String CONFIG_USERS_PATH = "/config/users"; + + + private final String BROKER_IDS_PATH = "/brokers/ids"; + private final String BROKER_TOPICS_PATH = "/brokers/topics"; + private final String TOPIC_ISR = "/brokers/topics/%s/partitions/%s/state"; + private final Logger LOG = LoggerFactory.getLogger(ConfigServiceImpl.class); + + + /** + * Get kafka broker numbers from zookeeper. + * @return + */ + @Override + public long getUserList(String clusterAlias) { + long count = 0; + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(CONFIG_USERS_PATH)) { + Seq path = zkc.getChildren(CONFIG_USERS_PATH); + List l = JavaConversions.seqAsJavaList(path); + count = l.size(); + } + } catch (Exception e) { + LOG.error("Get kafka broker numbers has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + + return count; + } + + + /** + * Instance Kafka Zookeeper client pool. + */ + private KafkaZKPoolUtils kafkaZKPool = KafkaZKPoolUtils.getInstance(); + + /** + * Kafka service interface. + */ + private KafkaService kafkaService = new KafkaFactory().create(); + + /** + * Statistics topic total used as page. + */ + public long topicNumbers(String clusterAlias) { + return topicList(clusterAlias).size(); + } + + /** + * Exclude kafka topic(__consumer_offsets) and blacklist topic. + */ + private void excludeTopic(String clusterAlias, List topics) { + if (topics.contains(Kafka.CONSUMER_OFFSET_TOPIC)) { + topics.remove(Kafka.CONSUMER_OFFSET_TOPIC); + } + String[] blackListTopics = SystemConfigUtils.getPropertyArray(clusterAlias + ".kafka.eagle.blacklist.topics", ","); + if (!StrUtils.isNull(blackListTopics)) { + for (String blackListTopic : blackListTopics) { + if (topics.contains(blackListTopic)) { + topics.remove(blackListTopic); + } + } + } + } + + /** + * Get search topic list numbers. + */ + public long topicNumbers(String clusterAlias, String topic) { + long count = 0L; + List topics = topicList(clusterAlias); + for (String name : topics) { + if (topic != null && name.contains(topic)) { + count++; + } + } + return count; + } + + /** + * Statistics topic partitions total used as page. + */ + public long partitionNumbers(String clusterAlias, String topic) { + long count = 0L; + if (Kafka.CONSUMER_OFFSET_TOPIC.equals(topic)) { + return count; + } + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic + "/partitions")) { + Seq subBrokerTopicsPaths = zkc.getChildren(BROKER_TOPICS_PATH + "/" + topic + "/partitions"); + count = JavaConversions.seqAsJavaList(subBrokerTopicsPaths).size(); + } + } catch (Exception e) { + LOG.error("Get topic partition numbers has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + return count; + } + + /** + * Get the number of page records for topic. + */ + public List topicRecords(String clusterAlias, Map params) { + List targets = new ArrayList(); + List topics = topicList(clusterAlias); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + int start = Integer.parseInt(params.get("start").toString()); + int length = Integer.parseInt(params.get("length").toString()); + if (params.containsKey("search") && params.get("search").toString().length() > 0) { + String search = params.get("search").toString(); + int offset = 0; + int id = start + 1; + for (String topic : topics) { + if (search != null && topic.contains(search)) { + if (offset < (start + length) && offset >= start) { + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + PartitionsInfo partition = new PartitionsInfo(); + partition.setId(id++); + partition.setCreated(CalendarUtils.convertUnixTime2Date(tuple._2.getCtime())); + partition.setModify(CalendarUtils.convertUnixTime2Date(tuple._2.getMtime())); + partition.setTopic(topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + partition.setPartitionNumbers(partitionObject.size()); + partition.setPartitions(partitionObject.keySet()); + targets.add(partition); + } + } catch (Exception ex) { + ex.printStackTrace(); + LOG.error("Scan topic search from zookeeper has error, msg is " + ex.getMessage()); + } + } + offset++; + } + } + } else { + int offset = 0; + int id = start + 1; + for (String topic : topics) { + if (offset < (start + length) && offset >= start) { + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + PartitionsInfo partition = new PartitionsInfo(); + partition.setId(id++); + partition.setCreated(CalendarUtils.convertUnixTime2Date(tuple._2.getCtime())); + partition.setModify(CalendarUtils.convertUnixTime2Date(tuple._2.getMtime())); + partition.setTopic(topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + partition.setPartitionNumbers(partitionObject.size()); + partition.setPartitions(partitionObject.keySet()); + targets.add(partition); + } + } catch (Exception ex) { + ex.printStackTrace(); + LOG.error("Scan topic page from zookeeper has error, msg is " + ex.getMessage()); + } + } + offset++; + } + } + } catch (Exception e) { + LOG.error("Get topic records has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return targets; + } + + /** + * Get broker spread by topic. + */ + public int getBrokerSpreadByTopic(String clusterAlias, String topic) { + int spread = 0; + try { + List topicMetas = topicMetadata(clusterAlias, topic); + Set brokerSizes = new HashSet<>(); + for (MetadataInfo meta : topicMetas) { + List replicasIntegers = new ArrayList<>(); + try { + replicasIntegers = JSON.parseObject(meta.getReplicas(), new TypeReference>() { + }); + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Parse string to int list has error, msg is " + e.getCause().getMessage()); + } + brokerSizes.addAll(replicasIntegers); + } + int brokerSize = kafkaService.getAllBrokersInfo(clusterAlias).size(); + spread = brokerSizes.size() * 100 / brokerSize; + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Get topic skewed info has error, msg is ", e); + } + return spread; + } + + /** + * Get broker skewed by topic. + */ + public int getBrokerSkewedByTopic(String clusterAlias, String topic) { + int skewed = 0; + try { + List topicMetas = topicMetadata(clusterAlias, topic); + int partitionAndReplicaTopics = 0; + Set brokerSizes = new HashSet<>(); + Map brokers = new HashMap<>(); + for (MetadataInfo meta : topicMetas) { + List replicasIntegers = new ArrayList<>(); + try { + replicasIntegers = JSON.parseObject(meta.getReplicas(), new TypeReference>() { + }); + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Parse string to int list has error, msg is " + e.getCause().getMessage()); + } + brokerSizes.addAll(replicasIntegers); + partitionAndReplicaTopics += replicasIntegers.size(); + for (Integer brokerId : replicasIntegers) { + if (brokers.containsKey(brokerId)) { + int value = brokers.get(brokerId); + brokers.put(brokerId, value + 1); + } else { + brokers.put(brokerId, 1); + } + } + } + int brokerSize = brokerSizes.size(); + int normalSkewedValue = MathUtils.ceil(brokerSize, partitionAndReplicaTopics); + int brokerSkewSize = 0; + for (Entry entry : brokers.entrySet()) { + if (entry.getValue() > normalSkewedValue) { + brokerSkewSize++; + } + } + skewed = brokerSkewSize * 100 / brokerSize; + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Get topic skewed info has error, msg is ", e); + } + return skewed; + } + + /** + * Get broker skewed leader by topic. + */ + public int getBrokerLeaderSkewedByTopic(String clusterAlias, String topic) { + int leaderSkewed = 0; + try { + List topicMetas = topicMetadata(clusterAlias, topic); + Map brokerLeaders = new HashMap<>(); + Set brokerSizes = new HashSet<>(); + for (MetadataInfo meta : topicMetas) { + List replicasIntegers = new ArrayList<>(); + try { + replicasIntegers = JSON.parseObject(meta.getReplicas(), new TypeReference>() { + }); + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Parse string to int list has error, msg is " + e.getCause().getMessage()); + } + brokerSizes.addAll(replicasIntegers); + if (brokerLeaders.containsKey(meta.getLeader())) { + int value = brokerLeaders.get(meta.getLeader()); + brokerLeaders.put(meta.getLeader(), value + 1); + } else { + brokerLeaders.put(meta.getLeader(), 1); + } + } + int brokerSize = brokerSizes.size(); + int brokerSkewLeaderNormal = MathUtils.ceil(brokerSize, topicMetas.size()); + int brokerSkewLeaderSize = 0; + for (Entry entry : brokerLeaders.entrySet()) { + if (entry.getValue() > brokerSkewLeaderNormal) { + brokerSkewLeaderSize++; + } + } + leaderSkewed = brokerSkewLeaderSize * 100 / brokerSize; + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Get topic skewed info has error, msg is ", e); + } + return leaderSkewed; + } + + /** + * Check topic from zookeeper metadata. + */ + public boolean findKafkaTopic(String clusterAlias, String topic) { + return topicList(clusterAlias).contains(topic); + } + + /** + * Get kafka broker numbers from zookeeper. + */ + public long brokerNumbers(String clusterAlias) { + long count = 0; + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_IDS_PATH)) { + Seq subBrokerIdsPaths = zkc.getChildren(BROKER_IDS_PATH); + count = JavaConversions.seqAsJavaList(subBrokerIdsPaths).size(); + } + } catch (Exception e) { + LOG.error("Get kafka broker numbers has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return count; + } + + /** + * Get topic list include cgroups from zookeeper. + */ + public List topicList(String clusterAlias) { + List topics = new ArrayList<>(); + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.cgroup.enable")) { + topics = SystemConfigUtils.getPropertyArrayList(clusterAlias + ".kafka.eagle.sasl.cgroup.topics", ","); + } else if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.cgroup.enable")) { + topics = SystemConfigUtils.getPropertyArrayList(clusterAlias + ".kafka.eagle.ssl.cgroup.topics", ","); + } else { + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH)) { + Seq subBrokerTopicsPaths = zkc.getChildren(BROKER_TOPICS_PATH); + topics = JavaConversions.seqAsJavaList(subBrokerTopicsPaths); + excludeTopic(clusterAlias, topics); + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get topic list has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + } + return topics; + } + + /** + * Get select topic list from zookeeper. + */ + public String topicListParams(String clusterAlias, String search) { + JSONArray targets = new JSONArray(); + int limit = 15; + + List topics = topicList(clusterAlias); + try { + if (Strings.isNullOrEmpty(search)) { + int id = 1; + for (String topic : topics) { + if (id <= limit) { + JSONObject object = new JSONObject(); + object.put("id", id); + object.put("name", topic); + targets.add(object); + id++; + } + } + } else { + int id = 1; + for (String topic : topics) { + if (topic.contains(search)) { + if (id <= limit) { + JSONObject object = new JSONObject(); + object.put("id", id); + object.put("name", topic); + targets.add(object); + id++; + } + } + } + } + } catch (Exception e) { + LOG.error("Get topic list has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } + return targets.toJSONString(); + } + + /** + * Scan topic meta page display from zookeeper and kafka. + */ + public List topicMetadataRecords(String clusterAlias, String topic, Map params) { + List targets = new ArrayList<>(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH)) { + List topics = topicList(clusterAlias); + if (topics.contains(topic)) { + int start = Integer.parseInt(params.get("start").toString()); + int length = Integer.parseInt(params.get("length").toString()); + int offset = 0; + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + Set partitionSet = new TreeSet<>(); + for (String partitionId : partitionObject.keySet()) { + partitionSet.add(Integer.valueOf(partitionId)); + } + Set partitionSortSet = new TreeSet<>(new Comparator() { + @Override + public int compare(Integer o1, Integer o2) { + int diff = o1 - o2;// asc + if (diff > 0) { + return 1; + } else if (diff < 0) { + return -1; + } + return 0; + } + }); + partitionSortSet.addAll(partitionSet); + for (int partition : partitionSortSet) { + if (offset < (start + length) && offset >= start) { + String path = String.format(TOPIC_ISR, topic, partition); + Tuple2, Stat> tuple2 = zkc.getDataAndStat(path); + String tupleString2 = new String(tuple2._1.get()); + JSONObject topicMetadata = JSON.parseObject(tupleString2); + MetadataInfo metadate = new MetadataInfo(); + metadate.setIsr(topicMetadata.getString("isr")); + metadate.setLeader(topicMetadata.getInteger("leader")); + metadate.setPartitionId(partition); + metadate.setReplicas(kafkaService.getReplicasIsr(clusterAlias, topic, partition)); + long logSize = 0L; + if ("kafka".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.offset.storage"))) { + logSize = kafkaService.getKafkaRealLogSize(clusterAlias, topic, partition); + } else { + logSize = kafkaService.getRealLogSize(clusterAlias, topic, partition); + } + List isrIntegers = new ArrayList<>(); + List replicasIntegers = new ArrayList<>(); + try { + isrIntegers = JSON.parseObject(metadate.getIsr(), new TypeReference>() { + }); + replicasIntegers = JSON.parseObject(metadate.getReplicas(), new TypeReference>() { + }); + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Parse string to int list has error, msg is ", e); + } + if (isrIntegers.size() != replicasIntegers.size()) { + // replicas lost + metadate.setUnderReplicated(true); + } else { + // replicas normal + metadate.setUnderReplicated(false); + } + if (replicasIntegers != null && replicasIntegers.size() > 0 && replicasIntegers.get(0) == metadate.getLeader()) { + // partition preferred leader + metadate.setPreferredLeader(true); + } else { + // partition occurs preferred leader exception + metadate.setPreferredLeader(false); + } + metadate.setLogSize(logSize); + targets.add(metadate); + } + offset++; + } + } + } + } catch (Exception e) { + LOG.error("Get topic metadata records has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return targets; + } + + private List topicMetadata(String clusterAlias, String topic) { + List targets = new ArrayList<>(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH)) { + List topics = topicList(clusterAlias); + if (topics.contains(topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + for (String partition : partitionObject.keySet()) { + String path = String.format(TOPIC_ISR, topic, Integer.valueOf(partition)); + Tuple2, Stat> tuple2 = zkc.getDataAndStat(path); + String tupleString2 = new String(tuple2._1.get()); + JSONObject topicMetadata = JSON.parseObject(tupleString2); + MetadataInfo metadate = new MetadataInfo(); + metadate.setIsr(topicMetadata.getString("isr")); + metadate.setLeader(topicMetadata.getInteger("leader")); + metadate.setPartitionId(Integer.valueOf(partition)); + metadate.setReplicas(kafkaService.getReplicasIsr(clusterAlias, topic, Integer.valueOf(partition))); + targets.add(metadate); + } + } + } + } catch (Exception e) { + LOG.error("Get topic metadata records has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return targets; + } + + /** + * Get topic producer logsize total. + */ + public long getTopicLogSizeTotal(String clusterAlias, String topic) { + long logSize = 0L; + if (Kafka.CONSUMER_OFFSET_TOPIC.equals(topic)) { + return logSize; + } + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + Set partitions = new HashSet<>(); + for (String partition : partitionObject.keySet()) { + try { + partitions.add(Integer.valueOf(partition)); + } catch (Exception e) { + LOG.error("Convert partition string to integer has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } + } + if ("kafka".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.offset.storage"))) { + logSize = kafkaService.getKafkaRealLogSize(clusterAlias, topic, partitions); + } else { + logSize = kafkaService.getLogSize(clusterAlias, topic, partitions); + } + } + } catch (Exception e) { + LOG.error("Get topic logsize total has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return logSize; + } + + /** + * Get topic real logsize records. + */ + public long getTopicRealLogSize(String clusterAlias, String topic) { + long logSize = 0L; + if (Kafka.CONSUMER_OFFSET_TOPIC.equals(topic)) { + return logSize; + } + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + Set partitions = new HashSet<>(); + for (String partition : partitionObject.keySet()) { + try { + partitions.add(Integer.valueOf(partition)); + } catch (Exception e) { + LOG.error("Convert partition string to integer has error, msg is ", e); + } + } + if ("kafka".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.offset.storage"))) { + logSize = kafkaService.getKafkaRealLogSize(clusterAlias, topic, partitions); + } else { + logSize = kafkaService.getRealLogSize(clusterAlias, topic, partitions); + } + } + } catch (Exception e) { + ErrorUtils.print(this.getClass()).error("Get topic real logsize has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return logSize; + } + + /** + * Get topic producer send logsize records. + */ + public long getTopicProducerLogSize(String clusterAlias, String topic) { + long logSize = 0L; + if (Kafka.CONSUMER_OFFSET_TOPIC.equals(topic)) { + return logSize; + } + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_TOPICS_PATH + "/" + topic)) { + Tuple2, Stat> tuple = zkc.getDataAndStat(BROKER_TOPICS_PATH + "/" + topic); + String tupleString = new String(tuple._1.get()); + JSONObject partitionObject = JSON.parseObject(tupleString).getJSONObject("partitions"); + Set partitions = new HashSet<>(); + for (String partition : partitionObject.keySet()) { + try { + partitions.add(Integer.valueOf(partition)); + } catch (Exception e) { + LOG.error("Convert partition string to integer has error, msg is " + e.getCause().getMessage()); + e.printStackTrace(); + } + } + if ("kafka".equals(SystemConfigUtils.getProperty(clusterAlias + ".kafka.eagle.offset.storage"))) { + logSize = kafkaService.getKafkaProducerLogSize(clusterAlias, topic, partitions); + } else { + logSize = kafkaService.getLogSize(clusterAlias, topic, partitions); + } + } + } catch (Exception e) { + LOG.error("Get topic real logsize has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return logSize; + } + + /** + * Add topic partitions. + */ + public Map createTopicPartitions(String clusterAlias, String topic, int totalCount) { + Map targets = new HashMap(); + int existPartitions = (int) partitionNumbers(clusterAlias, topic); + Properties prop = new Properties(); + prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaService.getKafkaBrokerServer(clusterAlias)); + + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { + kafkaService.sasl(prop, clusterAlias); + } + if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { + kafkaService.ssl(prop, clusterAlias); + } + + AdminClient adminClient = null; + try { + adminClient = AdminClient.create(prop); + Map newPartitions = new HashMap(); + newPartitions.put(topic, NewPartitions.increaseTo(existPartitions + totalCount)); + adminClient.createPartitions(newPartitions); + targets.put("status", "success"); + targets.put("info", "Add topic[" + topic + "], before partition[" + existPartitions + "], after partition[" + (existPartitions + totalCount) + "] has successed."); + } catch (Exception e) { + LOG.info("Add kafka topic partitions has error, msg is " + e.getMessage()); + e.printStackTrace(); + targets.put("status", "failed"); + targets.put("info", "Add kafka topic partitions has error, msg is " + e.getMessage()); + } finally { + adminClient.close(); + } + return targets; + } + + /** + * Get broker id list. + */ + public List getBrokerIdList(String clusterAlias) { + List brokerIds = new ArrayList<>(); + KafkaZkClient zkc = kafkaZKPool.getZkClient(clusterAlias); + try { + if (zkc.pathExists(BROKER_IDS_PATH)) { + Seq subBrokerIdsPaths = zkc.getChildren(BROKER_IDS_PATH); + for (String id : JavaConversions.seqAsJavaList(subBrokerIdsPaths)) { + brokerIds.add(Integer.parseInt(id)); + } + } + } catch (Exception e) { + LOG.error("Get kafka broker id has error, msg is ", e); + } finally { + if (zkc != null) { + kafkaZKPool.release(clusterAlias, zkc); + zkc = null; + } + } + return brokerIds; + } + +} diff --git a/kafka-eagle-plugin/.gitignore b/kafka-eagle-plugin/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/kafka-eagle-plugin/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/kafka-eagle-web/.gitignore b/kafka-eagle-web/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/kafka-eagle-web/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/AclController.java b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/AclController.java new file mode 100644 index 00000000..f57004e0 --- /dev/null +++ b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/AclController.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.web.controller; + +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.smartloli.kafka.eagle.common.util.KConstants; +import org.smartloli.kafka.eagle.core.factory.v2.ConfigFactory; +import org.smartloli.kafka.eagle.core.factory.v2.ConfigService; +import org.smartloli.kafka.eagle.web.service.AclService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.servlet.ModelAndView; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * Kafka acl controller to viewer data. + * + * @author jeff + * + */ +@Controller +public class AclController { + + /** Kafka acl service interface. */ + @Autowired + private AclService aclService; + + + /** + * Broker service interface. + */ + private static ConfigService configService = new ConfigFactory().create(); + + + + /** acls viewer. */ + @RequestMapping(value = "/acls", method = RequestMethod.GET) + public ModelAndView aclsView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + + + mav.setViewName("/acls/acls"); + return mav; + } + + + /** Get acl data by ajax. */ + @RequestMapping(value = "/acls/list", method = RequestMethod.GET) + @ResponseBody + public JSONArray aclslist(HttpServletResponse response, HttpServletRequest request) { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + JSONArray result = aclService.getAcls(clusterAlias); + configService.getUserList(clusterAlias); + return result; + } + + + @RequestMapping(value = "/acls/topic/{tname}", method = RequestMethod.GET) + @ResponseBody + public JSONArray aclstopic(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request, HttpSession session) { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + JSONArray result = aclService.getTopicAcls(clusterAlias, tname); + + return result; + } + + + /** + * Topic create viewer. + */ + @RequiresPermissions("/topic/create") + @RequestMapping(value = "/acls/createGroup", method = RequestMethod.GET) + public ModelAndView topicCreateView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/acls/createGroup"); + return mav; + } + + + /** Create topic form. */ + @RequestMapping(value = "/acls/createGroup/form", method = RequestMethod.POST) + public ModelAndView topicAddForm(HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + String ke_user_name = request.getParameter("ke_user_name"); + String ke_group_name = request.getParameter("ke_group_name"); + + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map respons = aclService.createGroup(clusterAlias, ke_user_name, ke_group_name); + + + + if ("success".equals(respons.get("status"))) { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/acls/create/success"); + } else { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/failed"); + } + return mav; + } + + + /** + * Create topic success viewer. + */ + @RequestMapping(value = "/acls/create/success", method = RequestMethod.GET) + public ModelAndView successView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/acls/add_success"); + return mav; + } + + /** + * Create topic failed viewer. + */ + @RequestMapping(value = "/acls/create/failed", method = RequestMethod.GET) + public ModelAndView failedView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/acls/add_failed"); + return mav; + } + + + @RequestMapping(value = "/acls/delete", method = RequestMethod.POST) + @ResponseBody + public String delete(@RequestBody JSONObject ob, + HttpSession session) throws Exception { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + return aclService.delete(clusterAlias, ob); + } + +} diff --git a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java index 7bb3bbb4..c3a57694 100644 --- a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java +++ b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java @@ -651,28 +651,32 @@ public ModelAndView cleanTopicDataAjax(@PathVariable("topic") String topic, Http return mav; } - /** - * Create topic form. - */ - @RequestMapping(value = "/topic/create/form", method = RequestMethod.POST) - public ModelAndView topicAddForm(HttpSession session, HttpServletResponse response, HttpServletRequest request) { - ModelAndView mav = new ModelAndView(); - String ke_topic_name = request.getParameter("ke_topic_name"); - String ke_topic_partition = request.getParameter("ke_topic_partition"); - String ke_topic_repli = request.getParameter("ke_topic_repli"); - String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); - Map respons = kafkaService.create(clusterAlias, ke_topic_name, ke_topic_partition, ke_topic_repli); - if ("success".equals(respons.get("status"))) { - session.removeAttribute("Submit_Status"); - session.setAttribute("Submit_Status", respons.get("info")); - mav.setViewName("redirect:/topic/create/success"); - } else { - session.removeAttribute("Submit_Status"); - session.setAttribute("Submit_Status", respons.get("info")); - mav.setViewName("redirect:/topic/create/failed"); - } - return mav; - } + /** Create topic form. */ + @RequestMapping(value = "/topic/create/form", method = RequestMethod.POST) + public ModelAndView topicAddForm(HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + String ke_topic_name = request.getParameter("ke_topic_name"); + String ke_topic_partition = request.getParameter("ke_topic_partition"); + String ke_topic_repli = request.getParameter("ke_topic_repli"); + String ke_topic_username = request.getParameter("ke_topic_username"); + + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map respons = kafkaService.create(clusterAlias, ke_topic_name, ke_topic_partition, ke_topic_repli, ke_topic_username); + + + + if ("success".equals(respons.get("status"))) { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/success"); + } else { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/failed"); + } + return mav; + } /** * Delete topic. diff --git a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java.orig b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java.orig new file mode 100644 index 00000000..fd97d345 --- /dev/null +++ b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/controller/TopicController.java.orig @@ -0,0 +1,1663 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.web.controller; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.common.base.Strings; +import org.apache.commons.lang.StringUtils; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.smartloli.kafka.eagle.common.protocol.MetadataInfo; +import org.smartloli.kafka.eagle.common.protocol.PartitionsInfo; +import org.smartloli.kafka.eagle.common.protocol.topic.*; +import org.smartloli.kafka.eagle.common.util.CalendarUtils; +import org.smartloli.kafka.eagle.common.util.KConstants; +import org.smartloli.kafka.eagle.common.util.KConstants.BrokerSever; +import org.smartloli.kafka.eagle.common.util.KConstants.Kafka; +import org.smartloli.kafka.eagle.common.util.KConstants.Role; +import org.smartloli.kafka.eagle.common.util.KConstants.Topic; +import org.smartloli.kafka.eagle.common.util.SystemConfigUtils; +import org.smartloli.kafka.eagle.core.factory.KafkaFactory; +import org.smartloli.kafka.eagle.core.factory.KafkaService; +import org.smartloli.kafka.eagle.core.factory.v2.BrokerFactory; +import org.smartloli.kafka.eagle.core.factory.v2.BrokerService; +import org.smartloli.kafka.eagle.web.pojo.Signiner; +import org.smartloli.kafka.eagle.web.service.TopicService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.servlet.ModelAndView; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import java.util.*; + +/** + * Kafka topic controller to viewer data. + * + * @author smartloli. + *

+ * Created by Sep 6, 2016. + *

+ * Update by hexiang 20170216 + */ +@Controller +public class TopicController { + +<<<<<<< HEAD + /** Kafka topic service interface. */ + @Autowired + private TopicService topicService; + + /** Kafka service interface. */ + private KafkaService kafkaService = new KafkaFactory().create(); + + /** BrokerService interface. */ + private BrokerService brokerService = new BrokerFactory().create(); + + /** Topic create viewer. */ + @RequiresPermissions("/topic/create") + @RequestMapping(value = "/topic/create", method = RequestMethod.GET) + public ModelAndView topicCreateView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/create"); + return mav; + } + + /** Topic message viewer. */ + @RequiresPermissions("/topic/message") + @RequestMapping(value = "/topic/message", method = RequestMethod.GET) + public ModelAndView topicMessageView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/msg"); + return mav; + } + + /** Topic message manager. */ + @RequiresPermissions("/topic/manager") + @RequestMapping(value = "/topic/manager", method = RequestMethod.GET) + public ModelAndView topicManagerView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/manager"); + return mav; + } + + /** Topic mock viewer. */ + @RequiresPermissions("/topic/mock") + @RequestMapping(value = "/topic/mock", method = RequestMethod.GET) + public ModelAndView topicMockView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/mock"); + return mav; + } + + /** Topic list viewer. */ + @RequestMapping(value = "/topic/list", method = RequestMethod.GET) + public ModelAndView topicListView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/list"); + return mav; + } + + /** Topic metadata viewer. */ + @RequestMapping(value = "/topic/meta/{tname}/", method = RequestMethod.GET) + public ModelAndView topicMetaView(@PathVariable("tname") String tname, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + if (topicService.hasTopic(clusterAlias, tname)) { + mav.setViewName("/topic/topic_meta"); + } else { + mav.setViewName("/error/404"); + } + + return mav; + } + + /** Create topic success viewer. */ + @RequestMapping(value = "/topic/create/success", method = RequestMethod.GET) + public ModelAndView successView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/add_success"); + return mav; + } + + /** Create topic failed viewer. */ + @RequestMapping(value = "/topic/create/failed", method = RequestMethod.GET) + public ModelAndView failedView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/add_failed"); + return mav; + } + + /** Get topic metadata by ajax. */ + @RequestMapping(value = "/topic/meta/{tname}/ajax", method = RequestMethod.GET) + public void topicMetaAjax(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } + } + + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map map = new HashMap<>(); + map.put("start", iDisplayStart); + map.put("length", iDisplayLength); + long count = topicService.getPartitionNumbers(clusterAlias, tname); + List metadatas = topicService.metadata(clusterAlias, tname, map); + JSONArray aaDatas = new JSONArray(); + for (MetadataInfo metadata : metadatas) { + JSONObject object = new JSONObject(); + object.put("topic", tname); + object.put("partition", metadata.getPartitionId()); + object.put("logsize", metadata.getLogSize()); + object.put("leader", metadata.getLeader()); + object.put("replicas", metadata.getReplicas()); + object.put("isr", metadata.getIsr()); + if (metadata.isPreferredLeader()) { + object.put("preferred_leader", "true"); + } else { + object.put("preferred_leader", "false"); + } + if (metadata.isUnderReplicated()) { + object.put("under_replicated", "true"); + } else { + object.put("under_replicated", "false"); + } + aaDatas.add(object); + } + + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", count); + target.put("iTotalDisplayRecords", count); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get cluster data by ajax. */ + @RequestMapping(value = "/topic/meta/mbean/{tname}/ajax", method = RequestMethod.GET) + public void topicMetaMetricsAjax(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.getTopicMBean(clusterAlias, tname); + + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get cluster data by ajax. */ + @RequestMapping(value = "/topic/meta/jmx/{tname}/ajax", method = RequestMethod.GET) + public void topicMsgByJmxAjax(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.getTopicSizeAndCapacity(clusterAlias, tname); + + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get topic datasets by ajax. */ + @RequestMapping(value = "/topic/mock/list/ajax", method = RequestMethod.GET) + public void topicMockAjax(HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String name = request.getParameter("name"); + JSONObject object = new JSONObject(); + object.put("items", JSON.parseArray(topicService.mockTopics(clusterAlias, name))); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get topic datasets by ajax. */ + @RequestMapping(value = "/topic/manager/keys/ajax", method = RequestMethod.GET) + public void getTopicProperties(HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String name = request.getParameter("name"); + JSONObject object = new JSONObject(); + object.put("items", JSON.parseArray(topicService.getTopicProperties(clusterAlias, name))); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get topic datasets by ajax. */ + @RequestMapping(value = "/topic/manager/{type}/ajax", method = RequestMethod.GET) + public void alterTopicConfigAjax(@PathVariable("type") String type, HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String topic = request.getParameter("topic"); + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setName(topic); + topicConfig.setType(type.toUpperCase()); + if (KConstants.Topic.ADD.equals(topicConfig.getType())) { + String key = request.getParameter("key"); + String value = request.getParameter("value"); + ConfigEntry configEntry = new ConfigEntry(key, value); + topicConfig.setConfigEntry(configEntry); + } else if (KConstants.Topic.DELETE.equals(topicConfig.getType())) { + String key = request.getParameter("key"); + ConfigEntry configEntry = new ConfigEntry(key, ""); + topicConfig.setConfigEntry(configEntry); + } + JSONObject object = new JSONObject(); + object.put("result", topicService.changeTopicConfig(clusterAlias, topicConfig)); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Send mock data to topic. */ + @RequestMapping(value = "/topic/mock/send/message/topic/ajax", method = RequestMethod.POST) + public void topicMockSend(@RequestBody TopicMockMessage topicMockMessage, HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + JSONObject object = new JSONObject(); + object.put("status", topicService.mockSendMsg(clusterAlias, topicMockMessage.getTopic(), topicMockMessage.getMessage())); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get topic datasets by ajax. */ + @RequestMapping(value = "/topic/list/table/ajax", method = RequestMethod.GET) + public void topicListAjax(HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + String search = ""; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } else if ("sSearch".equals(param.getString("name"))) { + search = param.getString("value"); + } + } + + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Signiner signiner = (Signiner) session.getAttribute(KConstants.Login.SESSION_USER); + + Map map = new HashMap<>(); + map.put("search", search); + map.put("start", iDisplayStart); + map.put("length", iDisplayLength); + long count = 0L; + if (search != null && !"".equals(search)) { + count = topicService.getTopicNumbers(clusterAlias, search); + } else { + count = topicService.getTopicNumbers(clusterAlias); + } + List topics = topicService.list(clusterAlias, map); + JSONArray aaDatas = new JSONArray(); + for (PartitionsInfo partition : topics) { + JSONObject object = new JSONObject(); + object.put("id", partition.getId()); + object.put("topic", "" + partition.getTopic() + ""); + object.put("partitions", partition.getPartitionNumbers()); + try { + long brokerSpread = partition.getBrokersSpread(); + if (brokerSpread < Topic.TOPIC_BROKER_SPREAD_ERROR) { + object.put("brokerSpread", "" + brokerSpread + "%"); + } else if (brokerSpread >= Topic.TOPIC_BROKER_SPREAD_ERROR && brokerSpread < Topic.TOPIC_BROKER_SPREAD_NORMAL) { + object.put("brokerSpread", "" + brokerSpread + "%"); + } else if (brokerSpread >= Topic.TOPIC_BROKER_SPREAD_NORMAL) { + object.put("brokerSpread", "" + brokerSpread + "%"); + } else { + object.put("brokerSpread", "" + brokerSpread + "%"); + } + + long brokerSkewed = partition.getBrokersSkewed(); + if (brokerSkewed >= Topic.TOPIC_BROKER_SKEW_ERROR) { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } else if (brokerSkewed > Topic.TOPIC_BROKER_SKEW_NORMAL && brokerSkewed < Topic.TOPIC_BROKER_SKEW_ERROR) { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } else if (brokerSkewed <= Topic.TOPIC_BROKER_SKEW_NORMAL) { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } else { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } + + long brokerLeaderSkewed = partition.getBrokersLeaderSkewed(); + if (brokerLeaderSkewed >= Topic.TOPIC_BROKER_LEADER_SKEW_ERROR) { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } else if (brokerLeaderSkewed > Topic.TOPIC_BROKER_LEADER_SKEW_NORMAL && brokerLeaderSkewed < Topic.TOPIC_BROKER_LEADER_SKEW_ERROR) { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } else if (brokerLeaderSkewed <= Topic.TOPIC_BROKER_LEADER_SKEW_NORMAL) { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } else { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } + } catch (Exception e) { + e.printStackTrace(); + } + object.put("created", partition.getCreated()); + object.put("modify", partition.getModify()); + Map topicStateParams = new HashMap<>(); + topicStateParams.put("cluster", clusterAlias); + topicStateParams.put("topic", partition.getTopic()); + topicStateParams.put("tkey", Topic.TRUNCATE); + List topicStates = topicService.getCleanTopicState(topicStateParams); + if (topicStates != null && topicStates.size() > 0) { + if (topicStates.get(0).getTvalue() == 0) { + if (Role.ADMIN.equals(signiner.getUsername())) { + object.put("operate", "

"); + } else { + object.put("operate", ""); + } + } else { + if (Role.ADMIN.equals(signiner.getUsername())) { + object.put("operate", "
"); + } else { + object.put("operate", ""); + } + } + } else { + if (Role.ADMIN.equals(signiner.getUsername())) { + object.put("operate", "
"); + } else { + object.put("operate", ""); + } + } + + aaDatas.add(object); + } + + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", count); + target.put("iTotalDisplayRecords", count); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get select topic datasets by ajax. */ + @RequestMapping(value = "/topic/list/select/ajax", method = RequestMethod.POST) + public void topicSelectListAjax(HttpServletResponse response, HttpServletRequest request) { + try { + String topic = request.getParameter("topic"); + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + byte[] output = topicService.getSelectTopics(clusterAlias, topic).getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get select filter topic datasets by ajax. */ + @RequestMapping(value = "/topic/list/filter/select/ajax", method = RequestMethod.GET) + public void topicSelectFilterListAjax(HttpServletResponse response, HttpServletRequest request) { + try { + String topics = request.getParameter("topics"); + String stime = request.getParameter("stime"); + String etime = request.getParameter("etime"); + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map params = new HashMap<>(); + params.put("cluster", clusterAlias); + params.put("stime", stime); + params.put("etime", etime); + if (!Strings.isNullOrEmpty(topics)) { + String[] topicStrs = topics.split(","); + Set topicSets = new HashSet<>(); + for (String topic : topicStrs) { + topicSets.add(topic); + } + params.put("topics", topicSets); + } + + byte[] output = topicService.getSelectTopicsLogSize(clusterAlias, params).getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Clean topic data by ajax. */ + @RequestMapping(value = "/topic/clean/data/{topic}/", method = RequestMethod.GET) + public ModelAndView cleanTopicDataAjax(@PathVariable("topic") String topic, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + TopicRank tr = new TopicRank(); + tr.setCluster(clusterAlias); + tr.setTopic(topic); + tr.setTkey(Topic.TRUNCATE); + tr.setTvalue(0); + if (topicService.addCleanTopicData(Arrays.asList(tr)) > 0) { + mav.setViewName("redirect:/topic/list"); + } else { + mav.setViewName("redirect:/errors/500"); + } + } catch (Exception ex) { + ex.printStackTrace(); + mav.setViewName("redirect:/errors/500"); + } + return mav; + } + + /** Create topic form. */ + @RequestMapping(value = "/topic/create/form", method = RequestMethod.POST) + public ModelAndView topicAddForm(HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + String ke_topic_name = request.getParameter("ke_topic_name"); + String ke_topic_partition = request.getParameter("ke_topic_partition"); + String ke_topic_repli = request.getParameter("ke_topic_repli"); + String ke_topic_username = request.getParameter("ke_topic_username"); + + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map respons = kafkaService.create(clusterAlias, ke_topic_name, ke_topic_partition, ke_topic_repli, ke_topic_username); + + + + if ("success".equals(respons.get("status"))) { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/success"); + } else { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/failed"); + } + return mav; + } + + /** Delete topic. */ + @RequestMapping(value = "/topic/{topicName}/{token}/delete", method = RequestMethod.GET) + public ModelAndView topicDelete(@PathVariable("topicName") String topicName, @PathVariable("token") String token, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + if (SystemConfigUtils.getProperty("kafka.eagle.topic.token").equals(token) && !Kafka.CONSUMER_OFFSET_TOPIC.equals(topicName)) { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Map respons = kafkaService.delete(clusterAlias, topicName); + if ("success".equals(respons.get("status"))) { + mav.setViewName("redirect:/topic/list"); + } else { + mav.setViewName("redirect:/errors/500"); + } + } else { + mav.setViewName("redirect:/errors/403"); + } + return mav; + } + + /** Modify topic partitions. */ + @RequestMapping(value = "/topic/{topicName}/{partitions}/modify", method = RequestMethod.GET) + public ModelAndView topicModifyPartitions(@PathVariable("topicName") String topicName, @PathVariable("partitions") int token, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Map respons = brokerService.createTopicPartitions(clusterAlias, topicName, token); + if ("success".equals(respons.get("status"))) { + mav.setViewName("redirect:/topic/list"); + } else { + mav.setViewName("redirect:/errors/500"); + } + return mav; + } + + /** Logical execute kafka sql. */ + @RequestMapping(value = "/topic/logical/commit/", method = RequestMethod.GET) + public void topicSqlLogicalAjax(@RequestParam String sql, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + TopicSqlHistory topicSql = new TopicSqlHistory(); + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.execute(clusterAlias, sql); + JSONObject result = JSON.parseObject(target); + try { + topicSql.setCluster(clusterAlias); + topicSql.setCreated(CalendarUtils.getDate()); + topicSql.setHost(request.getRemoteHost()); + topicSql.setKsql(sql); + if (result.getBoolean("error")) { + topicSql.setStatus("FAILED"); + topicSql.setSpendTime(0); + } else { + topicSql.setStatus("SUCCESSED"); + topicSql.setSpendTime(result.getLongValue("spent")); + } + topicSql.setTm(CalendarUtils.getCustomDate("yyyyMMdd")); + Signiner signin = (Signiner) SecurityUtils.getSubject().getSession().getAttribute(KConstants.Login.SESSION_USER); + topicSql.setUsername(signin.getUsername()); + topicService.writeTopicSqlHistory(Arrays.asList(topicSql)); + } catch (Exception e) { + e.printStackTrace(); + } + byte[] output = result.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get topic page message from kafka. */ + @RequestMapping(value = "/topic/physics/commit/", method = RequestMethod.GET) + public void topicSqlPhysicsAjax(@RequestParam String sql, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } + } + + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + String text = topicService.execute(clusterAlias, sql); + JSONObject result = JSON.parseObject(text); + + JSONArray topics = JSON.parseArray(result.getString("msg")); + JSONArray aaDatas = new JSONArray(); + int offset = 0; + if (topics != null) { + for (Object object : topics) { + JSONObject topic = (JSONObject) object; + if (offset < (iDisplayLength + iDisplayStart) && offset >= iDisplayStart) { + JSONObject obj = new JSONObject(); + for (String key : topic.keySet()) { + obj.put(key, topic.get(key)); + } + aaDatas.add(obj); + } + offset++; + } + } + + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", topics.size()); + target.put("iTotalDisplayRecords", topics.size()); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get topic sql history. */ + @RequestMapping(value = "/topic/sql/history/ajax", method = RequestMethod.GET) + public void topicSqlHistoryAjax(HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + String search = ""; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } else if ("sSearch".equals(param.getString("name"))) { + search = param.getString("value"); + } + } + + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Signiner signin = (Signiner) SecurityUtils.getSubject().getSession().getAttribute(KConstants.Login.SESSION_USER); + Map map = new HashMap(); + map.put("cluster", clusterAlias); + map.put("search", search); + map.put("username", signin.getUsername()); + map.put("start", iDisplayStart); + map.put("size", iDisplayLength); + long count = 0L; + List topicSqls = null; + if (signin.getUsername().equals(KConstants.Role.ADMIN)) { + topicSqls = topicService.readTopicSqlHistoryByAdmin(map); + count = topicService.countTopicSqlHistoryByAdmin(map); + } else { + topicSqls = topicService.readTopicSqlHistory(map); + count = topicService.countTopicSqlHistory(map); + } + + JSONArray aaDatas = new JSONArray(); + if (topicSqls != null) { + for (TopicSqlHistory topicSql : topicSqls) { + JSONObject obj = new JSONObject(); + int id = topicSql.getId(); + String host = topicSql.getHost(); + String ksql = topicSql.getKsql(); + obj.put("id", id); + obj.put("username", topicSql.getUsername()); + obj.put("host", "" + (host.length() > 20 ? host.substring(0, 20) + "..." : host) + ""); + obj.put("ksql", "" + (ksql.length() > 60 ? ksql.substring(0, 60) + "..." : ksql) + ""); + if (topicSql.getStatus().equals("SUCCESSED")) { + obj.put("status", "" + topicSql.getStatus() + ""); + } else { + obj.put("status", "" + topicSql.getStatus() + ""); + } + obj.put("spendTime", topicSql.getSpendTime() / 1000.0 + "s"); + obj.put("created", topicSql.getCreated()); + aaDatas.add(obj); + } + } + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", count); + target.put("iTotalDisplayRecords", count); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get ksql host or sql detail. */ + @RequestMapping(value = "/topic/ksql/detail/{type}/{id}/ajax", method = RequestMethod.GET) + public void getKSqlDetailByIdAjax(@PathVariable("id") int id, @PathVariable("type") String type, HttpServletResponse response, HttpServletRequest request) { + try { + JSONObject object = new JSONObject(); + Map params = new HashMap<>(); + params.put("id", id); + if ("host".equals(type)) { + object.put("result", topicService.findTopicSqlByID(params).getHost()); + } else if ("ksql".equals(type)) { + object.put("result", topicService.findTopicSqlByID(params).getKsql()); + } + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** Get producer chart data by ajax. */ + @RequestMapping(value = "/topic/producer/chart/ajax", method = RequestMethod.GET) + public void topicProducerChartAjax(HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map param = new HashMap<>(); + param.put("cluster", clusterAlias); + param.put("stime", request.getParameter("stime")); + param.put("etime", request.getParameter("etime")); + param.put("topic", request.getParameter("topic")); + String target = topicService.queryTopicProducerChart(param); + if (StringUtils.isEmpty(target)) { + target = ""; + } + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } +======= + /** + * Kafka topic service interface. + */ + @Autowired + private TopicService topicService; + + /** + * Kafka service interface. + */ + private KafkaService kafkaService = new KafkaFactory().create(); + + /** + * BrokerService interface. + */ + private BrokerService brokerService = new BrokerFactory().create(); + + /** + * Topic create viewer. + */ + @RequiresPermissions("/topic/create") + @RequestMapping(value = "/topic/create", method = RequestMethod.GET) + public ModelAndView topicCreateView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/create"); + return mav; + } + + /** + * Topic message viewer. + */ + @RequiresPermissions("/topic/message") + @RequestMapping(value = "/topic/message", method = RequestMethod.GET) + public ModelAndView topicMessageView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/ksql"); + return mav; + } + + /** + * Topic message manager. + */ + @RequiresPermissions("/topic/manager") + @RequestMapping(value = "/topic/manager", method = RequestMethod.GET) + public ModelAndView topicManagerView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/manager"); + return mav; + } + + /** + * Topic mock viewer. + */ + @RequiresPermissions("/topic/mock") + @RequestMapping(value = "/topic/mock", method = RequestMethod.GET) + public ModelAndView topicMockView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/mock"); + return mav; + } + + /** + * Topic mock viewer. + */ + @RequiresPermissions("/topic/hub") + @RequestMapping(value = "/topic/hub", method = RequestMethod.GET) + public ModelAndView topicHubView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/hub"); + return mav; + } + + /** + * Topic list viewer. + */ + @RequestMapping(value = "/topic/list", method = RequestMethod.GET) + public ModelAndView topicListView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/list"); + return mav; + } + + /** + * Topic metadata viewer. + */ + @RequestMapping(value = "/topic/meta/{tname}/", method = RequestMethod.GET) + public ModelAndView topicMetaView(@PathVariable("tname") String tname, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + if (topicService.hasTopic(clusterAlias, tname)) { + mav.setViewName("/topic/topic_meta"); + } else { + mav.setViewName("/error/404"); + } + + return mav; + } + + /** + * Create topic success viewer. + */ + @RequestMapping(value = "/topic/create/success", method = RequestMethod.GET) + public ModelAndView successView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/add_success"); + return mav; + } + + /** + * Create topic failed viewer. + */ + @RequestMapping(value = "/topic/create/failed", method = RequestMethod.GET) + public ModelAndView failedView(HttpSession session) { + ModelAndView mav = new ModelAndView(); + mav.setViewName("/topic/add_failed"); + return mav; + } + + /** + * Get topic metadata by ajax. + */ + @RequestMapping(value = "/topic/meta/{tname}/ajax", method = RequestMethod.GET) + public void topicMetaAjax(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } + } + + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map map = new HashMap<>(); + map.put("start", iDisplayStart); + map.put("length", iDisplayLength); + long count = topicService.getPartitionNumbers(clusterAlias, tname); + List metadatas = topicService.metadata(clusterAlias, tname, map); + JSONArray aaDatas = new JSONArray(); + for (MetadataInfo metadata : metadatas) { + JSONObject object = new JSONObject(); + object.put("topic", tname); + object.put("partition", metadata.getPartitionId()); + object.put("logsize", metadata.getLogSize()); + object.put("leader", metadata.getLeader()); + object.put("replicas", metadata.getReplicas()); + object.put("isr", metadata.getIsr()); + if (metadata.isPreferredLeader()) { + object.put("preferred_leader", "true"); + } else { + object.put("preferred_leader", "false"); + } + if (metadata.isUnderReplicated()) { + object.put("under_replicated", "true"); + } else { + object.put("under_replicated", "false"); + } + aaDatas.add(object); + } + + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", count); + target.put("iTotalDisplayRecords", count); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get cluster data by ajax. + */ + @RequestMapping(value = "/topic/meta/mbean/{tname}/ajax", method = RequestMethod.GET) + public void topicMetaMetricsAjax(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.getTopicMBean(clusterAlias, tname); + + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get cluster data by ajax. + */ + @RequestMapping(value = "/topic/meta/jmx/{tname}/ajax", method = RequestMethod.GET) + public void topicMsgByJmxAjax(@PathVariable("tname") String tname, HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.getTopicSizeAndCapacity(clusterAlias, tname); + + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get cluster producer and total capacity by ajax. + */ + @RequestMapping(value = "/topic/list/total/jmx/ajax", method = RequestMethod.GET) + public void producersCapacityByJmxAjax(HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.getProducersCapacity(clusterAlias); + + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get topic datasets by ajax. + */ + @RequestMapping(value = "/topic/mock/list/ajax", method = RequestMethod.GET) + public void topicMockAjax(HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String name = request.getParameter("name"); + JSONObject object = new JSONObject(); + object.put("items", JSON.parseArray(topicService.mockTopics(clusterAlias, name))); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get topic datasets by ajax. + */ + @RequestMapping(value = "/topic/manager/keys/ajax", method = RequestMethod.GET) + public void getTopicProperties(HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String name = request.getParameter("name"); + JSONObject object = new JSONObject(); + object.put("items", JSON.parseArray(topicService.getTopicProperties(clusterAlias, name))); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Balance topics by ajax. + */ + @RequestMapping(value = "/topic/balance/generate/", method = RequestMethod.GET) + public void topicGenerateAjax(HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String type = request.getParameter("type"); + List topicList = new ArrayList<>(); + if (BrokerSever.BALANCE_SINGLE.equals(type)) { + String topics = request.getParameter("topics"); + for (String topic : topics.split(",")) { + topicList.add(topic); + } + } + byte[] output = topicService.getBalanceGenerate(clusterAlias, topicList, type).toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Execute topics by ajax. + */ + @RequestMapping(value = "/topic/balance/execute/ajax", method = RequestMethod.POST) + public void topicExecuteAjax(@RequestBody TopicBalanceJson topicBalanceJson, HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + byte[] output = topicService.setBalanceExecute(clusterAlias, topicBalanceJson.getJson()).getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Verify topics by ajax. + */ + @RequestMapping(value = "/topic/balance/verify/ajax", method = RequestMethod.GET) + public void topicVerifyAjax(HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String type = request.getParameter("type"); + List topicList = new ArrayList<>(); + if (BrokerSever.BALANCE_SINGLE.equals(type)) { + String topics = request.getParameter("topics"); + for (String topic : topics.split(",")) { + topicList.add(topic); + } + } + String result = ""; + if (topicService.getBalanceGenerate(clusterAlias, topicList, type).containsKey("current")) { + String reassignTopicsJson = topicService.getBalanceGenerate(clusterAlias, topicList, type).getString("current"); + result = topicService.setBalanceVerify(clusterAlias, reassignTopicsJson); + } else { + result = topicService.getBalanceGenerate(clusterAlias, topicList, type).toJSONString(); + } + byte[] output = result.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get topic datasets by ajax. + */ + @RequestMapping(value = "/topic/manager/{type}/ajax", method = RequestMethod.GET) + public void alterTopicConfigAjax(@PathVariable("type") String type, HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String topic = request.getParameter("topic"); + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setName(topic); + topicConfig.setType(type.toUpperCase()); + if (KConstants.Topic.ADD.equals(topicConfig.getType())) { + String key = request.getParameter("key"); + String value = request.getParameter("value"); + ConfigEntry configEntry = new ConfigEntry(key, value); + topicConfig.setConfigEntry(configEntry); + } else if (KConstants.Topic.DELETE.equals(topicConfig.getType())) { + String key = request.getParameter("key"); + ConfigEntry configEntry = new ConfigEntry(key, ""); + topicConfig.setConfigEntry(configEntry); + } + JSONObject object = new JSONObject(); + object.put("result", topicService.changeTopicConfig(clusterAlias, topicConfig)); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Send mock data to topic. + */ + @RequestMapping(value = "/topic/mock/send/message/topic/ajax", method = RequestMethod.POST) + public void topicMockSend(@RequestBody TopicMockMessage topicMockMessage, HttpServletResponse response, HttpServletRequest request) { + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + JSONObject object = new JSONObject(); + object.put("status", topicService.mockSendMsg(clusterAlias, topicMockMessage.getTopic(), topicMockMessage.getMessage())); + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get topic datasets by ajax. + */ + @RequestMapping(value = "/topic/list/table/ajax", method = RequestMethod.GET) + public void topicListAjax(HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + String search = ""; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } else if ("sSearch".equals(param.getString("name"))) { + search = param.getString("value"); + } + } + + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Signiner signiner = (Signiner) session.getAttribute(KConstants.Login.SESSION_USER); + + Map map = new HashMap<>(); + map.put("search", search); + map.put("start", iDisplayStart); + map.put("length", iDisplayLength); + long count = 0L; + if (search != null && !"".equals(search)) { + count = topicService.getTopicNumbers(clusterAlias, search); + } else { + count = topicService.getTopicNumbers(clusterAlias); + } + List topics = topicService.list(clusterAlias, map); + JSONArray aaDatas = new JSONArray(); + for (PartitionsInfo partition : topics) { + JSONObject object = new JSONObject(); + object.put("id", partition.getId()); + object.put("topic", "" + partition.getTopic() + ""); + object.put("partitions", partition.getPartitionNumbers()); + try { + long brokerSpread = partition.getBrokersSpread(); + if (brokerSpread < Topic.TOPIC_BROKER_SPREAD_ERROR) { + object.put("brokerSpread", "" + brokerSpread + "%"); + } else if (brokerSpread >= Topic.TOPIC_BROKER_SPREAD_ERROR && brokerSpread < Topic.TOPIC_BROKER_SPREAD_NORMAL) { + object.put("brokerSpread", "" + brokerSpread + "%"); + } else if (brokerSpread >= Topic.TOPIC_BROKER_SPREAD_NORMAL) { + object.put("brokerSpread", "" + brokerSpread + "%"); + } else { + object.put("brokerSpread", "" + brokerSpread + "%"); + } + + long brokerSkewed = partition.getBrokersSkewed(); + if (brokerSkewed >= Topic.TOPIC_BROKER_SKEW_ERROR) { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } else if (brokerSkewed > Topic.TOPIC_BROKER_SKEW_NORMAL && brokerSkewed < Topic.TOPIC_BROKER_SKEW_ERROR) { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } else if (brokerSkewed <= Topic.TOPIC_BROKER_SKEW_NORMAL) { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } else { + object.put("brokerSkewed", "" + brokerSkewed + "%"); + } + + long brokerLeaderSkewed = partition.getBrokersLeaderSkewed(); + if (brokerLeaderSkewed >= Topic.TOPIC_BROKER_LEADER_SKEW_ERROR) { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } else if (brokerLeaderSkewed > Topic.TOPIC_BROKER_LEADER_SKEW_NORMAL && brokerLeaderSkewed < Topic.TOPIC_BROKER_LEADER_SKEW_ERROR) { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } else if (brokerLeaderSkewed <= Topic.TOPIC_BROKER_LEADER_SKEW_NORMAL) { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } else { + object.put("brokerLeaderSkewed", "" + brokerLeaderSkewed + "%"); + } + } catch (Exception e) { + e.printStackTrace(); + } + object.put("created", partition.getCreated()); + object.put("modify", partition.getModify()); + Map topicStateParams = new HashMap<>(); + topicStateParams.put("cluster", clusterAlias); + topicStateParams.put("topic", partition.getTopic()); + topicStateParams.put("tkey", Topic.TRUNCATE); + List topicStates = topicService.getCleanTopicState(topicStateParams); + if (topicStates != null && topicStates.size() > 0) { + if (topicStates.get(0).getTvalue() == 0) { + if (Role.ADMIN.equals(signiner.getUsername())) { + object.put("operate", + "
AlterDropTruncating
"); + } else { + object.put("operate", ""); + } + } else { + if (Role.ADMIN.equals(signiner.getUsername())) { + object.put("operate", + "
AlterDropTruncating
"); + } else { + object.put("operate", ""); + } + } + } else { + if (Role.ADMIN.equals(signiner.getUsername())) { + object.put("operate", + "
AlterDropTruncating
"); + } else { + object.put("operate", ""); + } + } + + aaDatas.add(object); + } + + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", count); + target.put("iTotalDisplayRecords", count); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get select topic datasets by ajax. + */ + @RequestMapping(value = "/topic/list/select/ajax", method = RequestMethod.POST) + public void topicSelectListAjax(HttpServletResponse response, HttpServletRequest request) { + try { + String topic = request.getParameter("topic"); + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + byte[] output = topicService.getSelectTopics(clusterAlias, topic).getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get select filter topic datasets by ajax. + */ + @RequestMapping(value = "/topic/list/filter/select/ajax", method = RequestMethod.GET) + public void topicSelectFilterListAjax(HttpServletResponse response, HttpServletRequest request) { + try { + String topics = request.getParameter("topics"); + String stime = request.getParameter("stime"); + String etime = request.getParameter("etime"); + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map params = new HashMap<>(); + params.put("cluster", clusterAlias); + params.put("stime", stime); + params.put("etime", etime); + if (!Strings.isNullOrEmpty(topics)) { + String[] topicStrs = topics.split(","); + Set topicSets = new HashSet<>(); + for (String topic : topicStrs) { + topicSets.add(topic); + } + params.put("topics", topicSets); + } + + byte[] output = topicService.getSelectTopicsLogSize(clusterAlias, params).getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Clean topic data by ajax. + */ + @RequestMapping(value = "/topic/clean/data/{topic}/", method = RequestMethod.GET) + public ModelAndView cleanTopicDataAjax(@PathVariable("topic") String topic, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + try { + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + TopicRank tr = new TopicRank(); + tr.setCluster(clusterAlias); + tr.setTopic(topic); + tr.setTkey(Topic.TRUNCATE); + tr.setTvalue(0); + if (topicService.addCleanTopicData(Arrays.asList(tr)) > 0) { + mav.setViewName("redirect:/topic/list"); + } else { + mav.setViewName("redirect:/errors/500"); + } + } catch (Exception ex) { + ex.printStackTrace(); + mav.setViewName("redirect:/errors/500"); + } + return mav; + } + + /** + * Create topic form. + */ + @RequestMapping(value = "/topic/create/form", method = RequestMethod.POST) + public ModelAndView topicAddForm(HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + String ke_topic_name = request.getParameter("ke_topic_name"); + String ke_topic_partition = request.getParameter("ke_topic_partition"); + String ke_topic_repli = request.getParameter("ke_topic_repli"); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Map respons = kafkaService.create(clusterAlias, ke_topic_name, ke_topic_partition, ke_topic_repli); + if ("success".equals(respons.get("status"))) { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/success"); + } else { + session.removeAttribute("Submit_Status"); + session.setAttribute("Submit_Status", respons.get("info")); + mav.setViewName("redirect:/topic/create/failed"); + } + return mav; + } + + /** + * Delete topic. + */ + @RequestMapping(value = "/topic/{topicName}/{token}/delete", method = RequestMethod.GET) + public ModelAndView topicDelete(@PathVariable("topicName") String topicName, @PathVariable("token") String token, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + if (SystemConfigUtils.getProperty("kafka.eagle.topic.token").equals(token) && !Kafka.CONSUMER_OFFSET_TOPIC.equals(topicName)) { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Map respons = kafkaService.delete(clusterAlias, topicName); + if ("success".equals(respons.get("status"))) { + mav.setViewName("redirect:/topic/list"); + } else { + mav.setViewName("redirect:/errors/500"); + } + } else { + mav.setViewName("redirect:/errors/403"); + } + return mav; + } + + /** + * Modify topic partitions. + */ + @RequestMapping(value = "/topic/{topicName}/{partitions}/modify", method = RequestMethod.GET) + public ModelAndView topicModifyPartitions(@PathVariable("topicName") String topicName, @PathVariable("partitions") int token, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + ModelAndView mav = new ModelAndView(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + Map respons = brokerService.createTopicPartitions(clusterAlias, topicName, token); + if ("success".equals(respons.get("status"))) { + mav.setViewName("redirect:/topic/list"); + } else { + mav.setViewName("redirect:/errors/500"); + } + return mav; + } + + /** + * Realtime access to shard sub scan task execute log. + */ + @RequestMapping(value = "/shard/sub/scan/log/", method = RequestMethod.GET) + public void getShardSubScanLogAjax(@RequestParam String jobId, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + try { + String logs = topicService.getShardLogs(jobId); + JSONObject object = new JSONObject(); + object.put("logs", logs); + byte[] output = object.toString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Logical execute kafka sql. + */ + @RequestMapping(value = "/topic/logical/commit/", method = RequestMethod.GET) + public void topicSqlLogicalAjax(@RequestParam String sql, @RequestParam String jobId, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + String target = topicService.execute(clusterAlias, sql, jobId, KConstants.Protocol.KSQL_LOGICAL); + JSONObject result = JSON.parseObject(target); + try { + TopicSqlHistory topicSql = new TopicSqlHistory(); + if (result.getBoolean("error")) { + topicSql.setStatus("FAILED"); + topicSql.setSpendTime(0); + topicSql.setCluster(clusterAlias); + topicSql.setCreated(CalendarUtils.getDate()); + topicSql.setHost(request.getRemoteHost()); + topicSql.setKsql(sql); + topicSql.setTm(CalendarUtils.getCustomDate("yyyyMMdd")); + Signiner signin = (Signiner) SecurityUtils.getSubject().getSession().getAttribute(KConstants.Login.SESSION_USER); + topicSql.setUsername(signin.getUsername()); + topicService.writeTopicSqlHistory(Arrays.asList(topicSql)); + } + } catch (Exception e) { + e.printStackTrace(); + } + byte[] output = result.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get topic page message from kafka. + */ + @RequestMapping(value = "/topic/physics/commit/", method = RequestMethod.GET) + public void topicSqlPhysicsAjax(@RequestParam String sql, @RequestParam String jobId, HttpSession session, HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } + } + + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + String text = topicService.execute(clusterAlias, sql, jobId, KConstants.Protocol.KSQL_PHYSICS); + JSONObject result = JSON.parseObject(text); + + try { + TopicSqlHistory topicSql = new TopicSqlHistory(); + topicSql.setCluster(clusterAlias); + topicSql.setCreated(CalendarUtils.getDate()); + topicSql.setHost(request.getRemoteHost()); + topicSql.setKsql(sql); + if (result.getBoolean("error")) { + topicSql.setStatus("FAILED"); + topicSql.setSpendTime(0); + } else { + topicSql.setStatus("SUCCESSED"); + topicSql.setSpendTime(result.getLongValue("spent")); + } + topicSql.setTm(CalendarUtils.getCustomDate("yyyyMMdd")); + Signiner signin = (Signiner) SecurityUtils.getSubject().getSession().getAttribute(KConstants.Login.SESSION_USER); + topicSql.setUsername(signin.getUsername()); + topicService.writeTopicSqlHistory(Arrays.asList(topicSql)); + } catch (Exception e) { + e.printStackTrace(); + } + + JSONArray topics = JSON.parseArray(result.getString("msg")); + JSONArray aaDatas = new JSONArray(); + int offset = 0; + if (topics != null) { + for (Object object : topics) { + JSONObject topic = (JSONObject) object; + if (offset < (iDisplayLength + iDisplayStart) && offset >= iDisplayStart) { + JSONObject obj = new JSONObject(); + for (String key : topic.keySet()) { + obj.put(key, topic.get(key)); + } + aaDatas.add(obj); + } + offset++; + } + } + + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", topics.size()); + target.put("iTotalDisplayRecords", topics.size()); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get topic sql history. + */ + @RequestMapping(value = "/topic/sql/history/ajax", method = RequestMethod.GET) + public void topicSqlHistoryAjax(HttpServletResponse response, HttpServletRequest request) { + String aoData = request.getParameter("aoData"); + JSONArray params = JSON.parseArray(aoData); + int sEcho = 0, iDisplayStart = 0, iDisplayLength = 0; + String search = ""; + for (Object object : params) { + JSONObject param = (JSONObject) object; + if ("sEcho".equals(param.getString("name"))) { + sEcho = param.getIntValue("value"); + } else if ("iDisplayStart".equals(param.getString("name"))) { + iDisplayStart = param.getIntValue("value"); + } else if ("iDisplayLength".equals(param.getString("name"))) { + iDisplayLength = param.getIntValue("value"); + } else if ("sSearch".equals(param.getString("name"))) { + search = param.getString("value"); + } + } + + HttpSession session = request.getSession(); + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Signiner signin = (Signiner) SecurityUtils.getSubject().getSession().getAttribute(KConstants.Login.SESSION_USER); + Map map = new HashMap(); + map.put("cluster", clusterAlias); + map.put("search", search); + map.put("username", signin.getUsername()); + map.put("start", iDisplayStart); + map.put("size", iDisplayLength); + long count = 0L; + List topicSqls = null; + if (signin.getUsername().equals(KConstants.Role.ADMIN)) { + topicSqls = topicService.readTopicSqlHistoryByAdmin(map); + count = topicService.countTopicSqlHistoryByAdmin(map); + } else { + topicSqls = topicService.readTopicSqlHistory(map); + count = topicService.countTopicSqlHistory(map); + } + + JSONArray aaDatas = new JSONArray(); + if (topicSqls != null) { + for (TopicSqlHistory topicSql : topicSqls) { + JSONObject obj = new JSONObject(); + int id = topicSql.getId(); + String host = topicSql.getHost(); + String ksql = topicSql.getKsql(); + obj.put("id", id); + obj.put("username", topicSql.getUsername()); + obj.put("host", "" + (host.length() > 20 ? host.substring(0, 20) + "..." : host) + ""); + obj.put("ksql", "" + (ksql.length() > 60 ? ksql.substring(0, 60) + "..." : ksql) + ""); + if (topicSql.getStatus().equals("SUCCESSED")) { + obj.put("status", "" + topicSql.getStatus() + ""); + } else { + obj.put("status", "" + topicSql.getStatus() + ""); + } + obj.put("spendTime", topicSql.getSpendTime() / 1000.0 + "s"); + obj.put("created", topicSql.getCreated()); + aaDatas.add(obj); + } + } + JSONObject target = new JSONObject(); + target.put("sEcho", sEcho); + target.put("iTotalRecords", count); + target.put("iTotalDisplayRecords", count); + target.put("aaData", aaDatas); + try { + byte[] output = target.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get ksql host or sql detail. + */ + @RequestMapping(value = "/topic/ksql/detail/{type}/{id}/ajax", method = RequestMethod.GET) + public void getKSqlDetailByIdAjax(@PathVariable("id") int id, @PathVariable("type") String type, HttpServletResponse response, HttpServletRequest request) { + try { + JSONObject object = new JSONObject(); + Map params = new HashMap<>(); + params.put("id", id); + if ("host".equals(type)) { + object.put("result", topicService.findTopicSqlByID(params).getHost()); + } else if ("ksql".equals(type)) { + object.put("result", topicService.findTopicSqlByID(params).getKsql()); + } + byte[] output = object.toJSONString().getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Get producer chart data by ajax. + */ + @RequestMapping(value = "/topic/producer/chart/ajax", method = RequestMethod.GET) + public void topicProducerChartAjax(HttpServletResponse response, HttpServletRequest request, HttpSession session) { + try { + String clusterAlias = session.getAttribute(KConstants.SessionAlias.CLUSTER_ALIAS).toString(); + + Map param = new HashMap<>(); + param.put("cluster", clusterAlias); + param.put("stime", request.getParameter("stime")); + param.put("etime", request.getParameter("etime")); + param.put("topic", request.getParameter("topic")); + String target = topicService.queryTopicProducerChart(param); + if (StringUtils.isEmpty(target)) { + target = ""; + } + byte[] output = target.getBytes(); + BaseController.response(output, response); + } catch (Exception ex) { + ex.printStackTrace(); + } + } +>>>>>>> bb39f8fca4bf158986bf8928cd7ee702afef5d71 + +} diff --git a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/AclService.java b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/AclService.java new file mode 100644 index 00000000..32d0fd45 --- /dev/null +++ b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/AclService.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.web.service; + +import java.util.Map; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * Kafka acl data interface. + * + * @author jeff. + * + * + */ +public interface AclService { + + /** Get acl data interface. */ + public JSONArray getAcls(String clusterAlias); + + /** Get acl data interface. */ + public JSONArray getTopicAcls(String clusterAlias, String topicname); + + Map createGroup(String clusterAlias, String userName, String groupName); + + String delete(String clusterAlias, JSONObject jsonObject); + +} diff --git a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/AclServiceImpl.java b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/AclServiceImpl.java new file mode 100644 index 00000000..fdfad84b --- /dev/null +++ b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/AclServiceImpl.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartloli.kafka.eagle.web.service.impl; + +import java.util.Map; + +import org.smartloli.kafka.eagle.core.factory.KafkaFactory; +import org.smartloli.kafka.eagle.core.factory.KafkaService; +import org.smartloli.kafka.eagle.web.service.AclService; +import org.springframework.stereotype.Service; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * Kafka acl data interface, and set up the return data set. + * + * @author jeff. + * + */ +@Service +public class AclServiceImpl implements AclService { + + /** Kafka service interface. */ + private KafkaService kafkaService = new KafkaFactory().create(); + + /** Get acl data from kafka cluster. */ + public JSONArray getAcls(String clusterAlias) { + JSONArray activeTopics = kafkaService.getKafkaAcl(clusterAlias); + + return activeTopics; + } + + @Override + public JSONArray getTopicAcls(String clusterAlias, String topicname) { + JSONArray acls = kafkaService.getKafkaAclBYTopicName(clusterAlias, topicname); + + return acls; + } + + @Override + public Map createGroup(String clusterAlias, String userName, String groupName) { + return kafkaService.createGroup(clusterAlias, userName, "group_" + userName + "_" + groupName); + } + + @Override + public String delete(String clusterAlias, JSONObject jsonObject) { + return kafkaService.deleteAcl(clusterAlias, jsonObject); + } + + +} + + + diff --git a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/TopicServiceImpl.java b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/TopicServiceImpl.java index f9e9f1c5..3503ddc8 100644 --- a/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/TopicServiceImpl.java +++ b/kafka-eagle-web/src/main/java/org/smartloli/kafka/eagle/web/service/impl/TopicServiceImpl.java @@ -45,6 +45,8 @@ import org.smartloli.kafka.eagle.core.factory.hub.KafkaHubService; import org.smartloli.kafka.eagle.core.factory.v2.BrokerFactory; import org.smartloli.kafka.eagle.core.factory.v2.BrokerService; +import org.smartloli.kafka.eagle.core.factory.v2.ConfigFactory; +import org.smartloli.kafka.eagle.core.factory.v2.ConfigService; import org.smartloli.kafka.eagle.core.metrics.KafkaMetricsFactory; import org.smartloli.kafka.eagle.core.metrics.KafkaMetricsService; import org.smartloli.kafka.eagle.core.sql.execute.KafkaSqlParser; @@ -88,6 +90,7 @@ public class TopicServiceImpl implements TopicService { * Broker service interface. */ private static BrokerService brokerService = new BrokerFactory().create(); + /** * Mx4j service interface. diff --git a/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/acls.jsp b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/acls.jsp new file mode 100644 index 00000000..0c0f3b41 --- /dev/null +++ b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/acls.jsp @@ -0,0 +1,119 @@ +<%@ page language="java" contentType="text/html; charset=UTF-8" + pageEncoding="UTF-8" %> +<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c" %> + + + + + + + + + + + + Consumers - KafkaEagle + + + + + + + + +
+
+ +
+
+
+
+

Consumers

+
    +
  1. ACl
    2. +
  2. List
    3. +
+
+ + List all ACL. +
+ +
+
+
+
+ List +
+
+
+ + + + + + + + + + + + +
resourceTypenameprincipalhostoperationpermissionTypeOperate
+
+
+
+
+
+ + + +
+
+ +
+
+ + + + + + + diff --git a/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/add_failed.jsp b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/add_failed.jsp new file mode 100644 index 00000000..582cace7 --- /dev/null +++ b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/add_failed.jsp @@ -0,0 +1,40 @@ +<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%> +<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> + + + + + +Failed - KafkaEagle + + + + + +
+
+ +
+
+
+
+

Topic

+
    +
  1. ACL
    2. +
  2. Create
    3. +
+
+ + ${Submit_Status} Create Again + +
+
+
+ +
+
+ + + + + \ No newline at end of file diff --git a/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/add_success.jsp b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/add_success.jsp new file mode 100644 index 00000000..5752a409 --- /dev/null +++ b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/add_success.jsp @@ -0,0 +1,40 @@ +<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%> +<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> + + + + + +Success - KafkaEagle + + + + + +
+
+ +
+
+
+
+

Topic

+
    +
  1. ACL
    2. +
  2. CreateGroup
    3. +
+
+ + ${Submit_Status} View Details + +
+
+
+ +
+
+ + + + + \ No newline at end of file diff --git a/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/createGroup.jsp b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/createGroup.jsp new file mode 100644 index 00000000..4a0f504a --- /dev/null +++ b/kafka-eagle-web/src/main/webapp/WEB-INF/views/acls/createGroup.jsp @@ -0,0 +1,106 @@ +<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%> +<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%> + + + + + + + + + + + + +Topic Create - KafkaEagle + + + + + +
+
+ +
+
+
+
+

Topic

+
    +
  1. ACL
    2. +
  2. CreateGroup
    3. +
+
+ + Create a new kafka's topic. +
+ +
+
+
+
+ Topic Property +
+
+
+
+
+
+ + + +
+ +
+ + + +
+ + + +
+
+
+
+
+
+
+
+
+ +
+
+ + + + + + \ No newline at end of file diff --git a/kafka-eagle-web/src/main/webapp/WEB-INF/views/public/plus/navbar.jsp b/kafka-eagle-web/src/main/webapp/WEB-INF/views/public/plus/navbar.jsp index 51c8f1ef..939c2c28 100644 --- a/kafka-eagle-web/src/main/webapp/WEB-INF/views/public/plus/navbar.jsp +++ b/kafka-eagle-web/src/main/webapp/WEB-INF/views/public/plus/navbar.jsp @@ -45,6 +45,7 @@ class="fas fa-cube fa-sm fa-fw mr-1">Hub
+
+ + +
+ +
+ +
Consumers - + + + +
Performance