8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

1、调整KafkaZKDao位置；2、offset信息获取时，过滤掉无leader分区；3、调整验证ZK是否合法时的session超时时间

Browse Source
This commit is contained in:
zengqiao
2022-09-22 11:30:46 +08:00
parent 2b76358c8f
commit 725ac10c3d
27 changed files with 74 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java
View File

@@ -5,7 +5,6 @@ import com.alibaba.fastjson.TypeReference;
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -79,20 +78,6 @@ public class Broker implements Serializable {
return metadata;
}
public static Broker buildFrom(Long clusterPhyId, Integer brokerId, BrokerMetadata brokerMetadata) {
Broker metadata = new Broker();
metadata.setClusterPhyId(clusterPhyId);
metadata.setBrokerId(brokerId);
metadata.setHost(brokerMetadata.getHost());
metadata.setPort(brokerMetadata.getPort());
metadata.setJmxPort(brokerMetadata.getJmxPort());
metadata.setStartTimestamp(brokerMetadata.getTimestamp());
metadata.setRack(brokerMetadata.getRack());
metadata.setStatus(1);
metadata.setEndpointMap(brokerMetadata.getEndpointMap());
return metadata;
}
public static Broker buildFrom(BrokerPO brokerPO) {
Broker broker = ConvertUtil.obj2Obj(brokerPO, Broker.class);
String endpointMapStr = brokerPO.getEndpointMap();

4
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/MsgConstant.java
View File

@@ -52,6 +52,10 @@ public class MsgConstant {
/**************************************************** Partition ****************************************************/
public static String getPartitionNoLeader(Long clusterPhyId, String topicName) {
return String.format("集群ID:[%d] Topic名称:[%s] 所有分区NoLeader", clusterPhyId, topicName);
}
public static String getPartitionNotExist(Long clusterPhyId, String topicName) {
return String.format("集群ID:[%d] Topic名称:[%s] 存在非法的分区ID", clusterPhyId, topicName);
}

22
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/ControllerData.java
View File

@@ -1,22 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* @author zengqiao
* @date 19/4/22
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class ControllerData {
private Integer brokerid;
private Integer version;
private Long timestamp;
}

129
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/BrokerMetadata.java
View File

@@ -1,129 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import lombok.Data;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author zengqiao
* @date 19/4/3
*
* 存储Broker的元信息, 元信息对应的ZK节点是/brokers/ids/{brokerId}
* 节点结构:
* {
* "listener_security_protocol_map":{"SASL_PLAINTEXT":"SASL_PLAINTEXT"},
* "endpoints":["SASL_PLAINTEXT://10.179.162.202:9093"],
* "jmx_port":9999,
* "host":null,
* "timestamp":"1546632983233",
* "port":-1,
* "version":4,
* "rack": "CY"
* }
*
* {
* "listener_security_protocol_map":{"SASL_PLAINTEXT":"SASL_PLAINTEXT","PLAINTEXT":"PLAINTEXT"},
* "endpoints":["SASL_PLAINTEXT://10.179.162.202:9093","PLAINTEXT://10.179.162.202:9092"],
* "jmx_port":8099,
* "host":"10.179.162.202",
* "timestamp":"1628833925822",
* "port":9092,
* "version":4
* }
*
* {
* "listener_security_protocol_map":{"EXTERNAL":"SASL_PLAINTEXT","INTERNAL":"SASL_PLAINTEXT"},
* "endpoints":["EXTERNAL://10.179.162.202:7092","INTERNAL://10.179.162.202:7093"],
* "jmx_port":8099,
* "host":null,
* "timestamp":"1627289710439",
* "port":-1,
* "version":4
* }
*
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class BrokerMetadata implements Serializable {
private static final long serialVersionUID = 3918113492423375809L;
private List<String> endpoints;
// <EXTERNAL|INTERNAL, <ip, port>>
private Map<String, IpPortData> endpointMap;
private String host;
private Integer port;
@JsonProperty("jmx_port")
private Integer jmxPort;
private Integer version;
private Long timestamp;
private String rack;
@JsonIgnore
public String getExternalHost() {
if (!endpointMap.containsKey(KafkaConstant.EXTERNAL_KEY)) {
// external如果不存在，就返回host
return host;
}
return endpointMap.get(KafkaConstant.EXTERNAL_KEY).getIp();
}
@JsonIgnore
public String getInternalHost() {
if (!endpointMap.containsKey(KafkaConstant.INTERNAL_KEY)) {
// internal如果不存在，就返回host
return host;
}
return endpointMap.get(KafkaConstant.INTERNAL_KEY).getIp();
}
public static void parseAndUpdateBrokerMetadata(BrokerMetadata brokerMetadata) {
brokerMetadata.setEndpointMap(new HashMap<>());
if (brokerMetadata.getEndpoints().isEmpty()) {
return;
}
// example EXTERNAL://10.179.162.202:7092
for (String endpoint: brokerMetadata.getEndpoints()) {
int idx1 = endpoint.indexOf("://");
int idx2 = endpoint.lastIndexOf(":");
if (idx1 == -1 || idx2 == -1 || idx1 == idx2) {
continue;
}
String brokerHost = endpoint.substring(idx1 + "://".length(), idx2);
String brokerPort = endpoint.substring(idx2 + 1);
brokerMetadata.getEndpointMap().put(endpoint.substring(0, idx1), new IpPortData(brokerHost, brokerPort));
if (KafkaConstant.INTERNAL_KEY.equals(endpoint.substring(0, idx1))) {
// 优先使用internal的地址进行展示
brokerMetadata.setHost(brokerHost);
brokerMetadata.setPort(ConvertUtil.string2Integer(brokerPort));
}
if (null == brokerMetadata.getHost()) {
brokerMetadata.setHost(brokerHost);
brokerMetadata.setPort(ConvertUtil.string2Integer(brokerPort));
}
}
}
}

41
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionMap.java
View File

@@ -1,41 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* 根据/brokers/topics/topic的节点内容定义
* @author tukun
* @date 2015/11/10.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class PartitionMap implements Serializable {
/**
* 版本号
*/
private int version;
/**
* Map<PartitionId，副本所在的brokerId列表>
*/
private Map<Integer, List<Integer>> partitions;
public List<Integer> getPartitionAssignReplicas(Integer partitionId) {
if (partitions == null) {
return new ArrayList<>();
}
return partitions.getOrDefault(partitionId, new ArrayList<>());
}
}

55
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/PartitionState.java
View File

@@ -1,55 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
import java.util.List;
/**
* PartitionState信息，对应ZK：/brokers/topics/__consumer_offsets/partitions/0/state 节点
* 该节点的数据结构：
* "{\"controller_epoch\":7,\"leader\":2,\"version\":1,\"leader_epoch\":7,\"isr\":[2,0]}"
* @author tukun
* @date 2015/11/10.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class PartitionState implements Serializable {
/**
* partition id
*/
private Integer partitionId;
/**
* kafka集群中的中央控制器选举次数
*/
@JsonProperty("controller_epoch")
private Integer controllerEpoch;
/**
* Partition所属的leader broker编号
*/
private Integer leader;
/**
* partition的版本号
*/
private Integer version;
/**
* 该partition leader选举次数
*/
@JsonProperty("leader_epoch")
private Integer leaderEpoch;
/**
* 同步副本组brokerId列表
*/
private List<Integer> isr;
}

55
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/brokers/TopicMetadata.java
View File

@@ -1,55 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
import java.util.Set;
/**
* 存储Topic的元信息, 元信息对应的ZK节点是/brokers/topics/${topicName}
* @author zengqiao
* @date 19/4/3
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class TopicMetadata implements Serializable {
/**
* topic名称
*/
private String topic;
/**
* partition所在的Broker
*/
private PartitionMap partitionMap;
/**
* topic所在的broker, 由partitionMap获取得到
*/
private Set<Integer> brokerIdSet;
/**
* 副本数
*/
private Integer replicaNum;
/**
* 分区数
*/
private Integer partitionNum;
/**
* 修改节点的时间
*/
private Long modifyTime;
/**
* 创建节点的时间
*/
private Long createTime;
}

18
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationBaseData.java
View File

@@ -1,18 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* @author zengqiao
* @date 20/5/14
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class ConfigChangeNotificationBaseData {
protected Integer version;
}

25
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV1.java
View File

@@ -1,25 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* @author zengqiao
* @date 20/5/14
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class ConfigChangeNotificationDataV1 extends ConfigChangeNotificationBaseData {
public static final Integer CHANGE_DATA_VERSION = 1;
@JsonProperty("entity_type")
private String entityType;
@JsonProperty("entity_name")
private String entityName;
}

29
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigChangeNotificationDataV2.java
View File

@@ -1,29 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* @author zengqiao
* @date 20/5/14
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class ConfigChangeNotificationDataV2 extends ConfigChangeNotificationBaseData {
public static final Integer CHANGE_DATA_VERSION = 2;
@JsonProperty("entity_path")
private String entityPath;
public static ConfigChangeNotificationDataV2 getChangeData(String entityPath) {
ConfigChangeNotificationDataV2 changeData = new ConfigChangeNotificationDataV2();
changeData.setEntityPath(entityPath);
changeData.setVersion(CHANGE_DATA_VERSION);
return changeData;
}
}

22
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/zookeeper/znode/config/ConfigNodeData.java
View File

@@ -1,22 +0,0 @@
package com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* @author zengqiao
* @date 20/5/12
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class ConfigNodeData<T> {
public static final Integer CONFIGDATA_VERSION = 1;
private T config;
private Integer version;
}