1、调整KafkaZKDao位置;2、offset信息获取时,过滤掉无leader分区;3、调整验证ZK是否合法时的session超时时间

This commit is contained in:
zengqiao
2022-09-22 11:30:46 +08:00
parent 2b76358c8f
commit 725ac10c3d
27 changed files with 74 additions and 59 deletions

View File

@@ -5,7 +5,6 @@ import com.alibaba.fastjson.TypeReference;
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData; import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO; import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@@ -79,20 +78,6 @@ public class Broker implements Serializable {
return metadata; return metadata;
} }
public static Broker buildFrom(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) {
Broker metadata = new Broker();
metadata.setClusterPhyId(clusterPhyId);
metadata.setBrokerId(brokerId);
metadata.setHost(brokerMetadata.getHost());
metadata.setPort(brokerMetadata.getPort());
metadata.setJmxPort(brokerMetadata.getJmxPort());
metadata.setStartTimestamp(brokerMetadata.getTimestamp());
metadata.setRack(brokerMetadata.getRack());
metadata.setStatus(1);
metadata.setEndpointMap(brokerMetadata.getEndpointMap());
return metadata;
}
public static Broker buildFrom(BrokerPO brokerPO) { public static Broker buildFrom(BrokerPO brokerPO) {
Broker broker = ConvertUtil.obj2Obj(brokerPO, Broker.class); Broker broker = ConvertUtil.obj2Obj(brokerPO, Broker.class);
String endpointMapStr = brokerPO.getEndpointMap(); String endpointMapStr = brokerPO.getEndpointMap();

View File

@@ -52,6 +52,10 @@ public class MsgConstant {
/**************************************************** Partition ****************************************************/ /**************************************************** Partition ****************************************************/
public static String getPartitionNoLeader(Long clusterPhyId, String topicName) {
return String.format("集群ID:[%d] Topic名称:[%s] 所有分区NoLeader", clusterPhyId, topicName);
}
public static String getPartitionNotExist(Long clusterPhyId, String topicName) { public static String getPartitionNotExist(Long clusterPhyId, String topicName) {
return String.format("集群ID:[%d] Topic名称:[%s] 存在非法的分区ID", clusterPhyId, topicName); return String.format("集群ID:[%d] Topic名称:[%s] 存在非法的分区ID", clusterPhyId, topicName);
} }

View File

@@ -5,7 +5,7 @@ import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy; import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException; import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.KafkaZkClient; import kafka.zk.KafkaZkClient;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;

View File

@@ -7,7 +7,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache; import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
public abstract class AbstractZKHandler { public abstract class AbstractZKHandler {

View File

@@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.BrokerIdsZNode; import kafka.zk.BrokerIdsZNode;
import kafka.zookeeper.ZNodeChildChangeHandler; import kafka.zookeeper.ZNodeChildChangeHandler;

View File

@@ -8,11 +8,11 @@ import com.xiaojukeji.know.streaming.km.common.enums.KafkaConfigTypeEnum;
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum; import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationBaseData; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationBaseData;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV1; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationDataV1;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV2; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationDataV2;
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.ConfigEntityChangeNotificationZNode; import kafka.zk.ConfigEntityChangeNotificationZNode;
import kafka.zookeeper.ZNodeChildChangeHandler; import kafka.zookeeper.ZNodeChildChangeHandler;
import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.data.Stat;

View File

@@ -11,7 +11,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService; import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.ControllerZNode; import kafka.zk.ControllerZNode;
import kafka.zookeeper.ZNodeChangeHandler; import kafka.zookeeper.ZNodeChangeHandler;

View File

@@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil; import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService; import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.TopicsZNode; import kafka.zk.TopicsZNode;
import kafka.zookeeper.ZNodeChildChangeHandler; import kafka.zookeeper.ZNodeChildChangeHandler;

View File

@@ -24,7 +24,6 @@ import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistExcept
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap; import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil; import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService; import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
@@ -32,8 +31,7 @@ import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerDAO; import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerDAO;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.BrokerIdZNode;
import kafka.zk.BrokerIdsZNode; import kafka.zk.BrokerIdsZNode;
import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.admin.*;
import org.apache.kafka.common.Node; import org.apache.kafka.common.Node;
@@ -310,9 +308,7 @@ public class BrokerServiceImpl extends BaseVersionControlService implements Brok
List<String> brokerIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), BrokerIdsZNode.path(), false); List<String> brokerIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), BrokerIdsZNode.path(), false);
for (String brokerId: brokerIdList) { for (String brokerId: brokerIdList) {
BrokerMetadata metadata = kafkaZKDAO.getData(clusterPhy.getId(), BrokerIdZNode.path(Integer.valueOf(brokerId)), BrokerMetadata.class); brokerList.add(kafkaZKDAO.getBrokerMetadata(clusterPhy.getId(), Integer.valueOf(brokerId)));
BrokerMetadata.parseAndUpdateBrokerMetadata(metadata);
brokerList.add(Broker.buildFrom(clusterPhy.getId(), Integer.valueOf(brokerId), metadata));
} }
return Result.buildSuc(brokerList); return Result.buildSuc(brokerList);

View File

@@ -13,8 +13,8 @@ import com.xiaojukeji.know.streaming.km.common.enums.valid.ValidateKafkaAddressE
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterValidateService; import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterValidateService;
import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO; import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import com.xiaojukeji.know.streaming.km.persistence.zk.impl.KafkaZKDAOImpl; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.impl.KafkaZKDAOImpl;
import kafka.server.KafkaConfig; import kafka.server.KafkaConfig;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.admin.*;

View File

@@ -19,7 +19,7 @@ import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService; import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
import com.xiaojukeji.know.streaming.km.persistence.mysql.kafkacontroller.KafkaControllerDAO; import com.xiaojukeji.know.streaming.km.persistence.mysql.kafkacontroller.KafkaControllerDAO;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.admin.*;
import org.apache.kafka.common.Node; import org.apache.kafka.common.Node;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;

View File

@@ -21,14 +21,14 @@ import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException; import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils; import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionMap;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionState; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionState;
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService; import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaConsumerClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaConsumerClient;
import com.xiaojukeji.know.streaming.km.persistence.mysql.partition.PartitionDAO; import com.xiaojukeji.know.streaming.km.persistence.mysql.partition.PartitionDAO;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.TopicPartitionStateZNode; import kafka.zk.TopicPartitionStateZNode;
import kafka.zk.TopicPartitionsZNode; import kafka.zk.TopicPartitionsZNode;
import kafka.zk.TopicZNode; import kafka.zk.TopicZNode;
@@ -202,10 +202,22 @@ public class PartitionServiceImpl extends BaseVersionControlService implements P
@Override @Override
public Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp) { public Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp) {
Map<TopicPartition, OffsetSpec> topicPartitionOffsets = new HashMap<>(); Map<TopicPartition, OffsetSpec> topicPartitionOffsets = new HashMap<>();
this.listPartitionByTopic(clusterPhyId, topicName)
.stream() List<Partition> partitionList = this.listPartitionByTopic(clusterPhyId, topicName);
if (partitionList == null || partitionList.isEmpty()) {
// Topic不存在
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(clusterPhyId, topicName));
}
partitionList.stream()
.filter(item -> !item.getLeaderBrokerId().equals(KafkaConstant.NO_LEADER))
.forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec)); .forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec));
if (topicPartitionOffsets.isEmpty()) {
// 所有分区no-leader
return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FAILED, MsgConstant.getPartitionNoLeader(clusterPhyId, topicName));
}
try { try {
return (Result<Map<TopicPartition, Long>>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicName, topicPartitionOffsets, timestamp)); return (Result<Map<TopicPartition, Long>>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicName, topicPartitionOffsets, timestamp));
} catch (VCHandlerNotExistException e) { } catch (VCHandlerNotExistException e) {

View File

@@ -23,7 +23,7 @@ import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.controller.ReplicaAssignment; import kafka.controller.ReplicaAssignment;
import kafka.server.ConfigType; import kafka.server.ConfigType;
import kafka.zk.AdminZkClient; import kafka.zk.AdminZkClient;

View File

@@ -30,7 +30,7 @@ import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService; import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.server.ConfigType; import kafka.server.ConfigType;
import kafka.zk.AdminZkClient; import kafka.zk.AdminZkClient;
import kafka.zk.KafkaZkClient; import kafka.zk.KafkaZkClient;

View File

@@ -23,7 +23,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService; import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
import com.xiaojukeji.know.streaming.km.persistence.mysql.topic.TopicDAO; import com.xiaojukeji.know.streaming.km.persistence.mysql.topic.TopicDAO;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.zk.TopicsZNode; import kafka.zk.TopicsZNode;
import org.apache.kafka.clients.admin.*; import org.apache.kafka.clients.admin.*;
import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.TopicPartitionInfo;

View File

@@ -0,0 +1,4 @@
/**
* 读取Kafka在ZK中存储的数据的包
*/
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.persistence.zk; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service;
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker; import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController; import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.persistence.zk.impl; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.impl;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.didiglobal.logi.log.ILog; import com.didiglobal.logi.log.ILog;
@@ -11,11 +11,11 @@ import com.xiaojukeji.know.streaming.km.common.enums.topic.TopicTypeEnum;
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException; import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException; import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.ControllerData; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.ControllerData;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.BrokerMetadata;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionMap;
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient; import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO; import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
import kafka.utils.Json; import kafka.utils.Json;
import kafka.zk.*; import kafka.zk.*;
import kafka.zookeeper.AsyncResponse; import kafka.zookeeper.AsyncResponse;
@@ -46,14 +46,14 @@ public class KafkaZKDAOImpl implements KafkaZKDAO {
public Broker getBrokerMetadata(String zkAddress) throws KeeperException.NoNodeException, AdminOperateException { public Broker getBrokerMetadata(String zkAddress) throws KeeperException.NoNodeException, AdminOperateException {
ZooKeeper zooKeeper = null; ZooKeeper zooKeeper = null;
try { try {
zooKeeper = new ZooKeeper(zkAddress, 1000, watchedEvent -> logger.info(" receive event : " + watchedEvent.getType().name())); zooKeeper = new ZooKeeper(zkAddress, 3000, watchedEvent -> logger.info(" receive event : " + watchedEvent.getType().name()));
List<String> brokerIdList = this.getChildren(zooKeeper, BrokerIdsZNode.path()); List<String> brokerIdList = this.getChildren(zooKeeper, BrokerIdsZNode.path());
if (brokerIdList == null || brokerIdList.isEmpty()) { if (brokerIdList == null || brokerIdList.isEmpty()) {
return null; return null;
} }
BrokerMetadata brokerMetadata = this.getData(zooKeeper, BrokerIdZNode.path(Integer.parseInt(brokerIdList.get(0))), false, BrokerMetadata.class); BrokerMetadata brokerMetadata = this.getData(zooKeeper, BrokerIdZNode.path(Integer.parseInt(brokerIdList.get(0))), false, BrokerMetadata.class);
return Broker.buildFrom(null, Integer.valueOf(brokerIdList.get(0)), brokerMetadata); return this.convert2Broker(null, Integer.valueOf(brokerIdList.get(0)), brokerMetadata);
} catch (KeeperException.NoNodeException nne) { } catch (KeeperException.NoNodeException nne) {
logger.warn("method=getBrokerMetadata||zkAddress={}||errMsg=exception", zkAddress, nne); logger.warn("method=getBrokerMetadata||zkAddress={}||errMsg=exception", zkAddress, nne);
throw nne; throw nne;
@@ -79,7 +79,7 @@ public class KafkaZKDAOImpl implements KafkaZKDAO {
try { try {
BrokerMetadata metadata = this.getData(kafkaZkClient.currentZooKeeper(), BrokerIdZNode.path(brokerId), false, BrokerMetadata.class); BrokerMetadata metadata = this.getData(kafkaZkClient.currentZooKeeper(), BrokerIdZNode.path(brokerId), false, BrokerMetadata.class);
BrokerMetadata.parseAndUpdateBrokerMetadata(metadata); BrokerMetadata.parseAndUpdateBrokerMetadata(metadata);
return Broker.buildFrom(clusterPhyId, brokerId, metadata); return this.convert2Broker(clusterPhyId, brokerId, metadata);
} catch (KeeperException ke) { } catch (KeeperException ke) {
logger.error("method=getBrokerMetadata||clusterPhyId={}||brokerId={}||errMsg=exception", clusterPhyId, brokerId, ke); logger.error("method=getBrokerMetadata||clusterPhyId={}||brokerId={}||errMsg=exception", clusterPhyId, brokerId, ke);
throw ke; throw ke;
@@ -269,4 +269,18 @@ public class KafkaZKDAOImpl implements KafkaZKDAO {
byte[] bytes = zooKeeper.getData(path, addWatch, null); byte[] bytes = zooKeeper.getData(path, addWatch, null);
return JSON.parseObject(bytes, clazz); return JSON.parseObject(bytes, clazz);
} }
private Broker convert2Broker(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) {
Broker metadata = new Broker();
metadata.setClusterPhyId(clusterPhyId);
metadata.setBrokerId(brokerId);
metadata.setHost(brokerMetadata.getHost());
metadata.setPort(brokerMetadata.getPort());
metadata.setJmxPort(brokerMetadata.getJmxPort());
metadata.setStartTimestamp(brokerMetadata.getTimestamp());
metadata.setRack(brokerMetadata.getRack());
metadata.setStatus(1);
metadata.setEndpointMap(brokerMetadata.getEndpointMap());
return metadata;
}
} }

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

View File

@@ -1,4 +1,4 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config; package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;