mirror of
https://github.com/didi/KnowStreaming.git
synced 2025-12-24 11:52:08 +08:00
1、调整KafkaZKDao位置;2、offset信息获取时,过滤掉无leader分区;3、调整验证ZK是否合法时的session超时时间
This commit is contained in:
@@ -5,7 +5,6 @@ import com.alibaba.fastjson.TypeReference;
|
|||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
|
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
|
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.NoArgsConstructor;
|
import lombok.NoArgsConstructor;
|
||||||
@@ -79,20 +78,6 @@ public class Broker implements Serializable {
|
|||||||
return metadata;
|
return metadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Broker buildFrom(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) {
|
|
||||||
Broker metadata = new Broker();
|
|
||||||
metadata.setClusterPhyId(clusterPhyId);
|
|
||||||
metadata.setBrokerId(brokerId);
|
|
||||||
metadata.setHost(brokerMetadata.getHost());
|
|
||||||
metadata.setPort(brokerMetadata.getPort());
|
|
||||||
metadata.setJmxPort(brokerMetadata.getJmxPort());
|
|
||||||
metadata.setStartTimestamp(brokerMetadata.getTimestamp());
|
|
||||||
metadata.setRack(brokerMetadata.getRack());
|
|
||||||
metadata.setStatus(1);
|
|
||||||
metadata.setEndpointMap(brokerMetadata.getEndpointMap());
|
|
||||||
return metadata;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Broker buildFrom(BrokerPO brokerPO) {
|
public static Broker buildFrom(BrokerPO brokerPO) {
|
||||||
Broker broker = ConvertUtil.obj2Obj(brokerPO, Broker.class);
|
Broker broker = ConvertUtil.obj2Obj(brokerPO, Broker.class);
|
||||||
String endpointMapStr = brokerPO.getEndpointMap();
|
String endpointMapStr = brokerPO.getEndpointMap();
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ public class MsgConstant {
|
|||||||
|
|
||||||
/**************************************************** Partition ****************************************************/
|
/**************************************************** Partition ****************************************************/
|
||||||
|
|
||||||
|
public static String getPartitionNoLeader(Long clusterPhyId, String topicName) {
|
||||||
|
return String.format("集群ID:[%d] Topic名称:[%s] 所有分区NoLeader", clusterPhyId, topicName);
|
||||||
|
}
|
||||||
|
|
||||||
public static String getPartitionNotExist(Long clusterPhyId, String topicName) {
|
public static String getPartitionNotExist(Long clusterPhyId, String topicName) {
|
||||||
return String.format("集群ID:[%d] Topic名称:[%s] 存在非法的分区ID", clusterPhyId, topicName);
|
return String.format("集群ID:[%d] Topic名称:[%s] 存在非法的分区ID", clusterPhyId, topicName);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import com.didiglobal.logi.log.LogFactory;
|
|||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.KafkaZkClient;
|
import kafka.zk.KafkaZkClient;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
|
|||||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
|
|
||||||
|
|
||||||
public abstract class AbstractZKHandler {
|
public abstract class AbstractZKHandler {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum
|
|||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.BrokerIdsZNode;
|
import kafka.zk.BrokerIdsZNode;
|
||||||
import kafka.zookeeper.ZNodeChildChangeHandler;
|
import kafka.zookeeper.ZNodeChildChangeHandler;
|
||||||
|
|
||||||
|
|||||||
@@ -8,11 +8,11 @@ import com.xiaojukeji.know.streaming.km.common.enums.KafkaConfigTypeEnum;
|
|||||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationBaseData;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationBaseData;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV1;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationDataV1;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV2;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config.ConfigChangeNotificationDataV2;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.ConfigEntityChangeNotificationZNode;
|
import kafka.zk.ConfigEntityChangeNotificationZNode;
|
||||||
import kafka.zookeeper.ZNodeChildChangeHandler;
|
import kafka.zookeeper.ZNodeChildChangeHandler;
|
||||||
import org.apache.zookeeper.data.Stat;
|
import org.apache.zookeeper.data.Stat;
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
|
|||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.ControllerZNode;
|
import kafka.zk.ControllerZNode;
|
||||||
import kafka.zookeeper.ZNodeChangeHandler;
|
import kafka.zookeeper.ZNodeChangeHandler;
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum
|
|||||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.TopicsZNode;
|
import kafka.zk.TopicsZNode;
|
||||||
import kafka.zookeeper.ZNodeChildChangeHandler;
|
import kafka.zookeeper.ZNodeChildChangeHandler;
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistExcept
|
|||||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
|
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||||
@@ -32,8 +31,7 @@ import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
|
|||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerDAO;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.BrokerIdZNode;
|
|
||||||
import kafka.zk.BrokerIdsZNode;
|
import kafka.zk.BrokerIdsZNode;
|
||||||
import org.apache.kafka.clients.admin.*;
|
import org.apache.kafka.clients.admin.*;
|
||||||
import org.apache.kafka.common.Node;
|
import org.apache.kafka.common.Node;
|
||||||
@@ -310,9 +308,7 @@ public class BrokerServiceImpl extends BaseVersionControlService implements Brok
|
|||||||
|
|
||||||
List<String> brokerIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), BrokerIdsZNode.path(), false);
|
List<String> brokerIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), BrokerIdsZNode.path(), false);
|
||||||
for (String brokerId: brokerIdList) {
|
for (String brokerId: brokerIdList) {
|
||||||
BrokerMetadata metadata = kafkaZKDAO.getData(clusterPhy.getId(), BrokerIdZNode.path(Integer.valueOf(brokerId)), BrokerMetadata.class);
|
brokerList.add(kafkaZKDAO.getBrokerMetadata(clusterPhy.getId(), Integer.valueOf(brokerId)));
|
||||||
BrokerMetadata.parseAndUpdateBrokerMetadata(metadata);
|
|
||||||
brokerList.add(Broker.buildFrom(clusterPhy.getId(), Integer.valueOf(brokerId), metadata));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Result.buildSuc(brokerList);
|
return Result.buildSuc(brokerList);
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ import com.xiaojukeji.know.streaming.km.common.enums.valid.ValidateKafkaAddressE
|
|||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterValidateService;
|
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterValidateService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.impl.KafkaZKDAOImpl;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.impl.KafkaZKDAOImpl;
|
||||||
import kafka.server.KafkaConfig;
|
import kafka.server.KafkaConfig;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.kafka.clients.admin.*;
|
import org.apache.kafka.clients.admin.*;
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
|||||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.kafkacontroller.KafkaControllerDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.mysql.kafkacontroller.KafkaControllerDAO;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import org.apache.kafka.clients.admin.*;
|
import org.apache.kafka.clients.admin.*;
|
||||||
import org.apache.kafka.common.Node;
|
import org.apache.kafka.common.Node;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
|||||||
@@ -21,14 +21,14 @@ import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
|||||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
|
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionMap;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionState;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionState;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaConsumerClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaConsumerClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.partition.PartitionDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.mysql.partition.PartitionDAO;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.TopicPartitionStateZNode;
|
import kafka.zk.TopicPartitionStateZNode;
|
||||||
import kafka.zk.TopicPartitionsZNode;
|
import kafka.zk.TopicPartitionsZNode;
|
||||||
import kafka.zk.TopicZNode;
|
import kafka.zk.TopicZNode;
|
||||||
@@ -202,10 +202,22 @@ public class PartitionServiceImpl extends BaseVersionControlService implements P
|
|||||||
@Override
|
@Override
|
||||||
public Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp) {
|
public Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp) {
|
||||||
Map<TopicPartition, OffsetSpec> topicPartitionOffsets = new HashMap<>();
|
Map<TopicPartition, OffsetSpec> topicPartitionOffsets = new HashMap<>();
|
||||||
this.listPartitionByTopic(clusterPhyId, topicName)
|
|
||||||
.stream()
|
List<Partition> partitionList = this.listPartitionByTopic(clusterPhyId, topicName);
|
||||||
|
if (partitionList == null || partitionList.isEmpty()) {
|
||||||
|
// Topic不存在
|
||||||
|
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(clusterPhyId, topicName));
|
||||||
|
}
|
||||||
|
|
||||||
|
partitionList.stream()
|
||||||
|
.filter(item -> !item.getLeaderBrokerId().equals(KafkaConstant.NO_LEADER))
|
||||||
.forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec));
|
.forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec));
|
||||||
|
|
||||||
|
if (topicPartitionOffsets.isEmpty()) {
|
||||||
|
// 所有分区no-leader
|
||||||
|
return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FAILED, MsgConstant.getPartitionNoLeader(clusterPhyId, topicName));
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return (Result<Map<TopicPartition, Long>>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicName, topicPartitionOffsets, timestamp));
|
return (Result<Map<TopicPartition, Long>>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicName, topicPartitionOffsets, timestamp));
|
||||||
} catch (VCHandlerNotExistException e) {
|
} catch (VCHandlerNotExistException e) {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|||||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.controller.ReplicaAssignment;
|
import kafka.controller.ReplicaAssignment;
|
||||||
import kafka.server.ConfigType;
|
import kafka.server.ConfigType;
|
||||||
import kafka.zk.AdminZkClient;
|
import kafka.zk.AdminZkClient;
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
|||||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.server.ConfigType;
|
import kafka.server.ConfigType;
|
||||||
import kafka.zk.AdminZkClient;
|
import kafka.zk.AdminZkClient;
|
||||||
import kafka.zk.KafkaZkClient;
|
import kafka.zk.KafkaZkClient;
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
|||||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.topic.TopicDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.mysql.topic.TopicDAO;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.zk.TopicsZNode;
|
import kafka.zk.TopicsZNode;
|
||||||
import org.apache.kafka.clients.admin.*;
|
import org.apache.kafka.clients.admin.*;
|
||||||
import org.apache.kafka.common.TopicPartitionInfo;
|
import org.apache.kafka.common.TopicPartitionInfo;
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
/**
|
||||||
|
* 读取Kafka在ZK中存储的数据的包
|
||||||
|
*/
|
||||||
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.persistence.zk;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service;
|
||||||
|
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.persistence.zk.impl;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.impl;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.didiglobal.logi.log.ILog;
|
import com.didiglobal.logi.log.ILog;
|
||||||
@@ -11,11 +11,11 @@ import com.xiaojukeji.know.streaming.km.common.enums.topic.TopicTypeEnum;
|
|||||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.ControllerData;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.ControllerData;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.BrokerMetadata;
|
||||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers.PartitionMap;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
import com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.service.KafkaZKDAO;
|
||||||
import kafka.utils.Json;
|
import kafka.utils.Json;
|
||||||
import kafka.zk.*;
|
import kafka.zk.*;
|
||||||
import kafka.zookeeper.AsyncResponse;
|
import kafka.zookeeper.AsyncResponse;
|
||||||
@@ -46,14 +46,14 @@ public class KafkaZKDAOImpl implements KafkaZKDAO {
|
|||||||
public Broker getBrokerMetadata(String zkAddress) throws KeeperException.NoNodeException, AdminOperateException {
|
public Broker getBrokerMetadata(String zkAddress) throws KeeperException.NoNodeException, AdminOperateException {
|
||||||
ZooKeeper zooKeeper = null;
|
ZooKeeper zooKeeper = null;
|
||||||
try {
|
try {
|
||||||
zooKeeper = new ZooKeeper(zkAddress, 1000, watchedEvent -> logger.info(" receive event : " + watchedEvent.getType().name()));
|
zooKeeper = new ZooKeeper(zkAddress, 3000, watchedEvent -> logger.info(" receive event : " + watchedEvent.getType().name()));
|
||||||
List<String> brokerIdList = this.getChildren(zooKeeper, BrokerIdsZNode.path());
|
List<String> brokerIdList = this.getChildren(zooKeeper, BrokerIdsZNode.path());
|
||||||
if (brokerIdList == null || brokerIdList.isEmpty()) {
|
if (brokerIdList == null || brokerIdList.isEmpty()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
BrokerMetadata brokerMetadata = this.getData(zooKeeper, BrokerIdZNode.path(Integer.parseInt(brokerIdList.get(0))), false, BrokerMetadata.class);
|
BrokerMetadata brokerMetadata = this.getData(zooKeeper, BrokerIdZNode.path(Integer.parseInt(brokerIdList.get(0))), false, BrokerMetadata.class);
|
||||||
return Broker.buildFrom(null, Integer.valueOf(brokerIdList.get(0)), brokerMetadata);
|
return this.convert2Broker(null, Integer.valueOf(brokerIdList.get(0)), brokerMetadata);
|
||||||
} catch (KeeperException.NoNodeException nne) {
|
} catch (KeeperException.NoNodeException nne) {
|
||||||
logger.warn("method=getBrokerMetadata||zkAddress={}||errMsg=exception", zkAddress, nne);
|
logger.warn("method=getBrokerMetadata||zkAddress={}||errMsg=exception", zkAddress, nne);
|
||||||
throw nne;
|
throw nne;
|
||||||
@@ -79,7 +79,7 @@ public class KafkaZKDAOImpl implements KafkaZKDAO {
|
|||||||
try {
|
try {
|
||||||
BrokerMetadata metadata = this.getData(kafkaZkClient.currentZooKeeper(), BrokerIdZNode.path(brokerId), false, BrokerMetadata.class);
|
BrokerMetadata metadata = this.getData(kafkaZkClient.currentZooKeeper(), BrokerIdZNode.path(brokerId), false, BrokerMetadata.class);
|
||||||
BrokerMetadata.parseAndUpdateBrokerMetadata(metadata);
|
BrokerMetadata.parseAndUpdateBrokerMetadata(metadata);
|
||||||
return Broker.buildFrom(clusterPhyId, brokerId, metadata);
|
return this.convert2Broker(clusterPhyId, brokerId, metadata);
|
||||||
} catch (KeeperException ke) {
|
} catch (KeeperException ke) {
|
||||||
logger.error("method=getBrokerMetadata||clusterPhyId={}||brokerId={}||errMsg=exception", clusterPhyId, brokerId, ke);
|
logger.error("method=getBrokerMetadata||clusterPhyId={}||brokerId={}||errMsg=exception", clusterPhyId, brokerId, ke);
|
||||||
throw ke;
|
throw ke;
|
||||||
@@ -269,4 +269,18 @@ public class KafkaZKDAOImpl implements KafkaZKDAO {
|
|||||||
byte[] bytes = zooKeeper.getData(path, addWatch, null);
|
byte[] bytes = zooKeeper.getData(path, addWatch, null);
|
||||||
return JSON.parseObject(bytes, clazz);
|
return JSON.parseObject(bytes, clazz);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Broker convert2Broker(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) {
|
||||||
|
Broker metadata = new Broker();
|
||||||
|
metadata.setClusterPhyId(clusterPhyId);
|
||||||
|
metadata.setBrokerId(brokerId);
|
||||||
|
metadata.setHost(brokerMetadata.getHost());
|
||||||
|
metadata.setPort(brokerMetadata.getPort());
|
||||||
|
metadata.setJmxPort(brokerMetadata.getJmxPort());
|
||||||
|
metadata.setStartTimestamp(brokerMetadata.getTimestamp());
|
||||||
|
metadata.setRack(brokerMetadata.getRack());
|
||||||
|
metadata.setStatus(1);
|
||||||
|
metadata.setEndpointMap(brokerMetadata.getEndpointMap());
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.brokers;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
|
package com.xiaojukeji.know.streaming.km.persistence.kafka.zookeeper.znode.config;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
Reference in New Issue
Block a user