8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

合并Master分支

Browse Source
This commit is contained in:
ZQKC
2023-07-07 13:09:28 +08:00
parent 9f6882cf0d 42195c3180
commit b1aa12bfa5
108 changed files with 2418 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
km-common/pom.xml
View File

@@ -5,13 +5,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-common</artifactId>
<version>${km.revision}</version>
<version>${revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<version>${revision}</version>
</parent>
<properties>

18
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/connector/ConnectorCreateDTO.java
View File

@@ -1,12 +1,12 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotNull;
import java.util.Properties;
/**
@@ -14,15 +14,23 @@ import java.util.Properties;
* @date 2022-10-17
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
@NoArgsConstructor
@ApiModel(description = "创建Connector")
public class ConnectorCreateDTO extends ClusterConnectorDTO {
@NotNull(message = "configs不允许为空")
@ApiModelProperty(value = "配置", example = "")
@Deprecated
@ApiModelProperty(value = "配置, 优先使用config字段，3.5.0版本将删除该字段", example = "")
protected Properties configs;
public ConnectorCreateDTO(Long connectClusterId, String connectorName, Properties configs) {
@ApiModelProperty(value = "配置", example = "")
protected Properties config;
public ConnectorCreateDTO(Long connectClusterId, String connectorName, Properties config) {
super(connectClusterId, connectorName);
this.configs = configs;
this.config = config;
}
public Properties getSuitableConfig() {
return config != null? config: configs;
}
}

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/mm2/MirrorMakerCreateDTO.java
View File

@@ -40,7 +40,7 @@ public class MirrorMakerCreateDTO extends ConnectorCreateDTO {
targetKafkaProps = new Properties();
}
this.unifyData(this.configs, sourceKafkaClusterId, sourceBootstrapServers, sourceKafkaProps, targetKafkaClusterId, targetBootstrapServers, targetKafkaProps);
this.unifyData(this.getSuitableConfig(), sourceKafkaClusterId, sourceBootstrapServers, sourceKafkaProps, targetKafkaClusterId, targetBootstrapServers, targetKafkaProps);
if (heartbeatConnectorConfigs != null) {
this.unifyData(this.heartbeatConnectorConfigs, sourceKafkaClusterId, sourceBootstrapServers, sourceKafkaProps, targetKafkaClusterId, targetBootstrapServers, targetKafkaProps);

40
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/group/GroupOffsetDeleteDTO.java Normal file
View File

@@ -0,0 +1,40 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.group;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
/**
* 删除offset
* @author zengqiao
* @date 19/4/8
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class GroupOffsetDeleteDTO extends BaseDTO {
@Min(value = 0, message = "clusterPhyId不允许为null或者小于0")
@ApiModelProperty(value = "集群ID", example = "6")
private Long clusterPhyId;
@NotBlank(message = "groupName不允许为空")
@ApiModelProperty(value = "消费组名称", example = "g-know-streaming")
private String groupName;
@ApiModelProperty(value = "Topic名称，按照Topic纬度进行删除时需要传", example = "know-streaming")
protected String topicName;
@ApiModelProperty(value = "分区ID，按照分区纬度进行删除时需要传")
private Integer partitionId;
/**
* @see com.xiaojukeji.know.streaming.km.common.enums.group.DeleteGroupTypeEnum
*/
@NotNull(message = "deleteType不允许为空")
@ApiModelProperty(value = "删除类型", example = "0:group纬度，1：Topic纬度，2：Partition纬度")
private Integer deleteType;
}

6
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java
View File

@@ -4,6 +4,8 @@ package com.xiaojukeji.know.streaming.km.common.bean.entity.broker;
import com.alibaba.fastjson.TypeReference;
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.enums.jmx.JmxEnum;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
@@ -71,10 +73,10 @@ public class Broker implements Serializable {
metadata.setBrokerId(node.id());
metadata.setHost(node.host());
metadata.setPort(node.port());
metadata.setJmxPort(-1);
metadata.setJmxPort(JmxEnum.UNKNOWN.getPort());
metadata.setStartTimestamp(startTimestamp);
metadata.setRack(node.rack());
metadata.setStatus(1);
metadata.setStatus(Constant.ALIVE);
return metadata;
}

29
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/JmxAuthConfig.java Normal file
View File

@@ -0,0 +1,29 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.config;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.Serializable;
/**
* @author zengqiao
* @date 23/05/19
*/
@Data
@ApiModel(description = "Jmx配置")
public class JmxAuthConfig implements Serializable {
@ApiModelProperty(value="最大连接", example = "100")
protected Integer maxConn;
@ApiModelProperty(value="是否开启SSL，如果开始则username 与 token 必须非空", example = "false")
protected Boolean openSSL;
@ApiModelProperty(value="SSL情况下的username", example = "Ks-Km")
protected String username;
@ApiModelProperty(value="SSL情况下的token", example = "KsKmCCY19")
protected String token;
}

77
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/JmxConfig.java
View File

@@ -1,10 +1,12 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.config;
import com.xiaojukeji.know.streaming.km.common.bean.entity.jmx.ServerIdJmxPort;
import com.xiaojukeji.know.streaming.km.common.enums.jmx.JmxEnum;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
/**
* @author zengqiao
@@ -12,24 +14,69 @@ import java.io.Serializable;
*/
@Data
@ApiModel(description = "Jmx配置")
public class JmxConfig implements Serializable {
@ApiModelProperty(value="jmx端口", example = "8099")
public class JmxConfig extends JmxAuthConfig {
@ApiModelProperty(value="jmx端口，最低优先使用的端口", example = "8099")
private Integer jmxPort;
@ApiModelProperty(value="最大连接", example = "100")
private Integer maxConn;
@ApiModelProperty(value="是否开启SSL，如果开始则username 与 token 必须非空", example = "false")
private Boolean openSSL;
@ApiModelProperty(value="SSL情况下的username", example = "Ks-Km")
private String username;
@ApiModelProperty(value="SSL情况下的token", example = "KsKmCCY19")
private String token;
@ApiModelProperty(value="使用哪个endpoint网络", example = "EXTERNAL")
private String useWhichEndpoint;
@ApiModelProperty(value="指定server的JMX端口, 最高优先使用的端口", example = "")
private List<ServerIdJmxPort> specifiedJmxPortList;
/**
* 选取最终的jmx端口
* @param serverId 服务ID
* @param metadataJmxPort ks从元信息中获取到的jmx端口
*/
public Integer getFinallyJmxPort(String serverId, Integer metadataJmxPort) {
if (specifiedJmxPortList == null || specifiedJmxPortList.isEmpty()) {
// 未进行特殊指定时，zkJMX端口存在则优先使用zkJmxPort，否则使用配置的jmxPort
return this.selectJmxPort(jmxPort, metadataJmxPort);
}
// 进行特殊配置时
for (ServerIdJmxPort serverIdJmxPort: specifiedJmxPortList) {
if (serverId.equals(serverIdJmxPort.getServerId()) && serverIdJmxPort.getJmxPort() != null) {
// 当前server有指定具体的jmx端口时，则使用具体的端口
return serverIdJmxPort.getJmxPort();
}
}
return this.selectJmxPort(jmxPort, metadataJmxPort);
}
/**
* 选取最终的jmx端口
* @param serverId serverId
*/
public Integer getFinallyJmxPort(String serverId) {
return this.getFinallyJmxPort(serverId, null);
}
/**
* 选取jmx端口
* @param feJmxPort 前端页面配置的jmx端口
* @param metadataJmxPort ks从元信息中获取到的jmx端口
*/
private Integer selectJmxPort(Integer feJmxPort, Integer metadataJmxPort) {
if (metadataJmxPort == null) {
return feJmxPort != null? feJmxPort: JmxEnum.NOT_OPEN.getPort();
}
if (JmxEnum.NOT_OPEN.getPort().equals(metadataJmxPort)) {
// 如果元信息提示未开启，则直接返回未开启
return JmxEnum.NOT_OPEN.getPort();
}
if (JmxEnum.UNKNOWN.getPort().equals(metadataJmxPort)) {
// 如果元信息提示未知，则直接返回feJmxPort 或者 未开启
return feJmxPort != null? feJmxPort: JmxEnum.NOT_OPEN.getPort();
}
// 其他情况，返回 metadataJmxPort
return metadataJmxPort;
}
}

17
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/ConnectCluster.java
View File

@@ -1,6 +1,7 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.EntityIdInterface;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import lombok.Data;
import java.io.Serializable;
@@ -54,6 +55,22 @@ public class ConnectCluster implements Serializable, Comparable<ConnectCluster>,
*/
private String clusterUrl;
public String getSuitableRequestUrl() {
// 优先使用用户填写的url
String suitableRequestUrl = this.clusterUrl;
if (ValidateUtils.isBlank(suitableRequestUrl)) {
// 用户如果没有填写，则使用元信息中的url
suitableRequestUrl = this.memberLeaderUrl;
}
//url去斜杠
if (suitableRequestUrl.length() > 0 && suitableRequestUrl.charAt(suitableRequestUrl.length() - 1) == '/') {
return suitableRequestUrl.substring(0, suitableRequestUrl.length() - 1);
}
return suitableRequestUrl;
}
@Override
public int compareTo(ConnectCluster connectCluster) {
return this.id.compareTo(connectCluster.getId());

25
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/jmx/ServerIdJmxPort.java Normal file
View File

@@ -0,0 +1,25 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.jmx;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
/**
* @author didi
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ServerIdJmxPort implements Serializable {
/**
* serverID
*/
private String serverId;
/**
* JMX端口
*/
private Integer jmxPort;
}

4
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/BaseMetrics.java
View File

@@ -27,6 +27,10 @@ public abstract class BaseMetrics implements Serializable {
protected Map<String, Float> metrics = new ConcurrentHashMap<>();
public void putMetric(String key, Float value){
if (value == null || key == null) {
return;
}
metrics.put(key, value);
}

16
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/group/DeleteGroupParam.java Normal file
View File

@@ -0,0 +1,16 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.group;
import com.xiaojukeji.know.streaming.km.common.enums.group.DeleteGroupTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class DeleteGroupParam extends GroupParam {
protected DeleteGroupTypeEnum deleteGroupTypeEnum;
public DeleteGroupParam(Long clusterPhyId, String groupName, DeleteGroupTypeEnum deleteGroupTypeEnum) {
super(clusterPhyId, groupName);
this.deleteGroupTypeEnum = deleteGroupTypeEnum;
}
}

16
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/group/DeleteGroupTopicParam.java Normal file
View File

@@ -0,0 +1,16 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.group;
import com.xiaojukeji.know.streaming.km.common.enums.group.DeleteGroupTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class DeleteGroupTopicParam extends DeleteGroupParam {
protected String topicName;
public DeleteGroupTopicParam(Long clusterPhyId, String groupName, DeleteGroupTypeEnum deleteGroupTypeEnum, String topicName) {
super(clusterPhyId, groupName, deleteGroupTypeEnum);
this.topicName = topicName;
}
}

16
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/group/DeleteGroupTopicPartitionParam.java Normal file
View File

@@ -0,0 +1,16 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.group;
import com.xiaojukeji.know.streaming.km.common.enums.group.DeleteGroupTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class DeleteGroupTopicPartitionParam extends DeleteGroupTopicParam {
protected Integer partitionId;
public DeleteGroupTopicPartitionParam(Long clusterPhyId, String groupName, DeleteGroupTypeEnum deleteGroupTypeEnum, String topicName, Integer partitionId) {
super(clusterPhyId, groupName, deleteGroupTypeEnum, topicName);
this.partitionId = partitionId;
}
}

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/group/GroupParam.java
View File

@@ -1,13 +1,11 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.group;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class GroupParam extends ClusterPhyParam {
protected String groupName;

29
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/topic/TopicTruncateParam.java Normal file
View File

@@ -0,0 +1,29 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TopicTruncateParam extends ClusterPhyParam {
protected String topicName;
protected long offset;
public TopicTruncateParam(Long clusterPhyId, String topicName, long offset) {
super(clusterPhyId);
this.topicName = topicName;
this.offset = offset;
}
@Override
public String toString() {
return "TopicParam{" +
"clusterPhyId=" + clusterPhyId +
", topicName='" + topicName + '\'' +
", offset='" + offset + '\'' +
'}';
}
}

20
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/zookeeper/fourletterword/MonitorCmdData.java
View File

@@ -25,15 +25,15 @@ public class MonitorCmdData extends BaseFourLetterWordCmdData {
private Float zkAvgLatency;
private Float zkMaxLatency;
private Float zkMinLatency;
private Long zkPacketsReceived;
private Long zkPacketsSent;
private Long zkNumAliveConnections;
private Long zkOutstandingRequests;
private Float zkPacketsReceived;
private Float zkPacketsSent;
private Float zkNumAliveConnections;
private Float zkOutstandingRequests;
private String zkServerState;
private Long zkZnodeCount;
private Long zkWatchCount;
private Long zkEphemeralsCount;
private Long zkApproximateDataSize;
private Long zkOpenFileDescriptorCount;
private Long zkMaxFileDescriptorCount;
private Float zkZnodeCount;
private Float zkWatchCount;
private Float zkEphemeralsCount;
private Float zkApproximateDataSize;
private Float zkOpenFileDescriptorCount;
private Float zkMaxFileDescriptorCount;
}

10
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/zookeeper/fourletterword/ServerCmdData.java
View File

@@ -20,11 +20,11 @@ public class ServerCmdData extends BaseFourLetterWordCmdData {
private Float zkAvgLatency;
private Float zkMaxLatency;
private Float zkMinLatency;
private Long zkPacketsReceived;
private Long zkPacketsSent;
private Long zkNumAliveConnections;
private Long zkOutstandingRequests;
private Float zkPacketsReceived;
private Float zkPacketsSent;
private Float zkNumAliveConnections;
private Float zkOutstandingRequests;
private String zkServerState;
private Long zkZnodeCount;
private Float zkZnodeCount;
private Long zkZxid;
}

22
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/zookeeper/fourletterword/parser/MonitorCmdDataParser.java
View File

@@ -51,7 +51,7 @@ public class MonitorCmdDataParser implements FourLetterWordDataParser<MonitorCmd
}
MonitorCmdData monitorCmdData = new MonitorCmdData();
dataMap.entrySet().stream().forEach(elem -> {
dataMap.entrySet().forEach(elem -> {
try {
switch (elem.getKey()) {
case "zk_version":
@@ -67,37 +67,37 @@ public class MonitorCmdDataParser implements FourLetterWordDataParser<MonitorCmd
monitorCmdData.setZkMinLatency(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_packets_received":
monitorCmdData.setZkPacketsReceived(Long.valueOf(elem.getValue()));
monitorCmdData.setZkPacketsReceived(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_packets_sent":
monitorCmdData.setZkPacketsSent(Long.valueOf(elem.getValue()));
monitorCmdData.setZkPacketsSent(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_num_alive_connections":
monitorCmdData.setZkNumAliveConnections(Long.valueOf(elem.getValue()));
monitorCmdData.setZkNumAliveConnections(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_outstanding_requests":
monitorCmdData.setZkOutstandingRequests(Long.valueOf(elem.getValue()));
monitorCmdData.setZkOutstandingRequests(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_server_state":
monitorCmdData.setZkServerState(elem.getValue());
break;
case "zk_znode_count":
monitorCmdData.setZkZnodeCount(Long.valueOf(elem.getValue()));
monitorCmdData.setZkZnodeCount(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_watch_count":
monitorCmdData.setZkWatchCount(Long.valueOf(elem.getValue()));
monitorCmdData.setZkWatchCount(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_ephemerals_count":
monitorCmdData.setZkEphemeralsCount(Long.valueOf(elem.getValue()));
monitorCmdData.setZkEphemeralsCount(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_approximate_data_size":
monitorCmdData.setZkApproximateDataSize(Long.valueOf(elem.getValue()));
monitorCmdData.setZkApproximateDataSize(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_open_file_descriptor_count":
monitorCmdData.setZkOpenFileDescriptorCount(Long.valueOf(elem.getValue()));
monitorCmdData.setZkOpenFileDescriptorCount(ConvertUtil.string2Float(elem.getValue()));
break;
case "zk_max_file_descriptor_count":
monitorCmdData.setZkMaxFileDescriptorCount(Long.valueOf(elem.getValue()));
monitorCmdData.setZkMaxFileDescriptorCount(ConvertUtil.string2Float(elem.getValue()));
break;
case "Proposal sizes last/min/max":
case "zk_fsync_threshold_exceed_count":

12
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/zookeeper/fourletterword/parser/ServerCmdDataParser.java
View File

@@ -46,7 +46,7 @@ public class ServerCmdDataParser implements FourLetterWordDataParser<ServerCmdDa
}
ServerCmdData serverCmdData = new ServerCmdData();
dataMap.entrySet().stream().forEach(elem -> {
dataMap.entrySet().forEach(elem -> {
try {
switch (elem.getKey()) {
case "Zookeeper version":
@@ -59,22 +59,22 @@ public class ServerCmdDataParser implements FourLetterWordDataParser<ServerCmdDa
serverCmdData.setZkMaxLatency(ConvertUtil.string2Float(data[2]));
break;
case "Received":
serverCmdData.setZkPacketsReceived(Long.valueOf(elem.getValue()));
serverCmdData.setZkPacketsReceived(ConvertUtil.string2Float(elem.getValue()));
break;
case "Sent":
serverCmdData.setZkPacketsSent(Long.valueOf(elem.getValue()));
serverCmdData.setZkPacketsSent(ConvertUtil.string2Float(elem.getValue()));
break;
case "Connections":
serverCmdData.setZkNumAliveConnections(Long.valueOf(elem.getValue()));
serverCmdData.setZkNumAliveConnections(ConvertUtil.string2Float(elem.getValue()));
break;
case "Outstanding":
serverCmdData.setZkOutstandingRequests(Long.valueOf(elem.getValue()));
serverCmdData.setZkOutstandingRequests(ConvertUtil.string2Float(elem.getValue()));
break;
case "Mode":
serverCmdData.setZkServerState(elem.getValue());
break;
case "Node count":
serverCmdData.setZkZnodeCount(Long.valueOf(elem.getValue()));
serverCmdData.setZkZnodeCount(ConvertUtil.string2Float(elem.getValue()));
break;
case "Zxid":
serverCmdData.setZkZxid(Long.parseUnsignedLong(elem.getValue().trim().substring(2), 16));

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/connect/ConnectClusterPO.java
View File

@@ -29,7 +29,7 @@ public class ConnectClusterPO extends BasePO {
private Integer state;
/**
* 集群地址
* 用户填写的集群地址
*/
private String clusterUrl;

13
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/group/GroupMemberPO.java
View File

@@ -7,6 +7,7 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Date;
import java.util.Objects;
@Data
@NoArgsConstructor
@@ -37,4 +38,16 @@ public class GroupMemberPO extends BasePO {
this.memberCount = memberCount;
this.updateTime = updateTime;
}
public boolean equal2GroupMemberPO(GroupMemberPO that) {
if (that == null) {
return false;
}
return Objects.equals(clusterPhyId, that.clusterPhyId)
&& Objects.equals(topicName, that.topicName)
&& Objects.equals(groupName, that.groupName)
&& Objects.equals(state, that.state)
&& Objects.equals(memberCount, that.memberCount);
}
}

16
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/group/GroupPO.java
View File

@@ -9,6 +9,8 @@ import com.xiaojukeji.know.streaming.km.common.enums.group.GroupTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.Objects;
@Data
@NoArgsConstructor
@@ -58,4 +60,18 @@ public class GroupPO extends BasePO {
*/
private int coordinatorId;
public boolean equal2GroupPO(GroupPO groupPO) {
if (groupPO == null) {
return false;
}
return coordinatorId == groupPO.coordinatorId
&& Objects.equals(clusterPhyId, groupPO.clusterPhyId)
&& Objects.equals(type, groupPO.type)
&& Objects.equals(name, groupPO.name)
&& Objects.equals(state, groupPO.state)
&& Objects.equals(memberCount, groupPO.memberCount)
&& Objects.equals(topicMembers, groupPO.topicMembers)
&& Objects.equals(partitionAssignor, groupPO.partitionAssignor);
}
}

1
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/mm2/MirrorMakerBaseStateVO.java
View File

@@ -13,7 +13,6 @@ import lombok.Data;
@Data
@ApiModel(description = "集群MM2状态信息")
public class MirrorMakerBaseStateVO extends BaseVO {
@ApiModelProperty(value = "worker数", example = "1")
private Integer workerCount;

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/KafkaConstant.java
View File

@@ -49,6 +49,8 @@ public class KafkaConstant {
public static final Map<String, ConfigDef.ConfigKey> KAFKA_ALL_CONFIG_DEF_MAP = new ConcurrentHashMap<>();
public static final Integer TOPICK_TRUNCATE_DEFAULT_OFFSET = -1;
static {
try {
KAFKA_ALL_CONFIG_DEF_MAP.putAll(CollectionConverters.asJava(LogConfig$.MODULE$.configKeys()));

5
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/PaginationConstant.java
View File

@@ -27,5 +27,8 @@ public class PaginationConstant {
/**
* groupTopic列表的默认排序规则
*/
public static final String DEFAULT_GROUP_TOPIC_SORTED_FIELD = "topicName";
public static final String DEFAULT_GROUP_TOPIC_SORTED_FIELD = "topicName";
public static final String TOPIC_RECORDS_TIME_SORTED_FIELD = "timestampUnitMs";
public static final String TOPIC_RECORDS_OFFSET_SORTED_FIELD = "offset";
}

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/converter/GroupConverter.java
View File

@@ -10,6 +10,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.stream.Collectors;
/**
@@ -57,6 +58,7 @@ public class GroupConverter {
po.setTopicMembers(ConvertUtil.obj2Json(group.getTopicMembers()));
po.setType(group.getType().getCode());
po.setState(group.getState().getState());
po.setUpdateTime(new Date());
return po;
}
}

28
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/group/DeleteGroupTypeEnum.java Normal file
View File

@@ -0,0 +1,28 @@
package com.xiaojukeji.know.streaming.km.common.enums.group;
import lombok.Getter;
/**
* @author wyb
* @date 2022/10/11
*/
@Getter
public enum DeleteGroupTypeEnum {
UNKNOWN(-1, "Unknown"),
GROUP(0, "Group纬度"),
GROUP_TOPIC(1, "GroupTopic纬度"),
GROUP_TOPIC_PARTITION(2, "GroupTopicPartition纬度");
private final Integer code;
private final String msg;
DeleteGroupTypeEnum(Integer code, String msg) {
this.code = code;
this.msg = msg;
}
}

20
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/jmx/JmxEnum.java Normal file
View File

@@ -0,0 +1,20 @@
package com.xiaojukeji.know.streaming.km.common.enums.jmx;
import lombok.Getter;
@Getter
public enum JmxEnum {
NOT_OPEN(-1, "未开启JMX端口"),
UNKNOWN(-2, "JMX端口未知"),
;
private final Integer port;
private final String message;
JmxEnum(Integer port, String message) {
this.port = port;
this.message = message;
}
}

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/OperationEnum.java
View File

@@ -32,6 +32,8 @@ public enum OperationEnum {
RESTART(11, "重启"),
TRUNCATE(12, "清空"),
;
OperationEnum(int code, String desc) {

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/version/VersionItemTypeEnum.java
View File

@@ -41,6 +41,8 @@ public enum VersionItemTypeEnum {
SERVICE_OP_REASSIGNMENT(330, "service_reassign_operation"),
SERVICE_OP_GROUP(340, "service_group_operation"),
SERVICE_OP_CONNECT_CLUSTER(400, "service_connect_cluster_operation"),
SERVICE_OP_CONNECT_CONNECTOR(401, "service_connect_connector_operation"),
SERVICE_OP_CONNECT_PLUGIN(402, "service_connect_plugin_operation"),

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/zookeeper/ZKRoleEnum.java
View File

@@ -10,6 +10,8 @@ public enum ZKRoleEnum {
OBSERVER("observer"),
STANDALONE("standalone"),
UNKNOWN("unknown"),
;

56
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxConnectorWrap.java
View File

@@ -1,6 +1,8 @@
package com.xiaojukeji.know.streaming.km.common.jmx;
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxAuthConfig;
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig;
import com.xiaojukeji.know.streaming.km.common.enums.jmx.JmxEnum;
import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import org.slf4j.Logger;
@@ -33,26 +35,26 @@ public class JmxConnectorWrap {
private final Long brokerStartupTime;
private final String host;
private final String jmxHost;
private final Integer port;
private final Integer jmxPort;
private JMXConnector jmxConnector;
private final AtomicInteger atomicInteger;
private JmxConfig jmxConfig;
private JmxAuthConfig jmxConfig;
public JmxConnectorWrap(String clientLogIdent, Long brokerStartupTime, String host, Integer port, JmxConfig jmxConfig) {
this.clientLogIdent=clientLogIdent;
public JmxConnectorWrap(String clientLogIdent, Long brokerStartupTime, String jmxHost, Integer jmxPort, JmxAuthConfig jmxConfig) {
LOGGER.info(
"method=JmxConnectorWrap||clientLogIdent={}||brokerStartupTime={}||jmxHost={}||jmxPort={}||jmxConfig={}||msg=start construct JmxWrap.",
clientLogIdent, brokerStartupTime, jmxHost, jmxPort, jmxConfig
);
this.clientLogIdent = clientLogIdent;
this.brokerStartupTime = brokerStartupTime;
this.host = host;
if (port == null || port == -1 && jmxConfig.getJmxPort() != null) {
this.port = jmxConfig.getJmxPort();
} else {
this.port = port;
}
this.jmxHost = jmxHost;
this.jmxPort = (jmxPort == null? JmxEnum.UNKNOWN.getPort() : jmxPort);
this.jmxConfig = jmxConfig;
if (ValidateUtils.isNull(this.jmxConfig)) {
@@ -61,6 +63,7 @@ public class JmxConnectorWrap {
if (ValidateUtils.isNullOrLessThanZero(this.jmxConfig.getMaxConn())) {
this.jmxConfig.setMaxConn(1000);
}
this.atomicInteger = new AtomicInteger(this.jmxConfig.getMaxConn());
}
@@ -68,7 +71,7 @@ public class JmxConnectorWrap {
if (jmxConnector != null) {
return true;
}
if (port == null || port == -1) {
if (jmxPort == null || jmxPort == -1) {
return false;
}
return createJmxConnector();
@@ -91,7 +94,10 @@ public class JmxConnectorWrap {
jmxConnector = null;
} catch (IOException e) {
LOGGER.warn("close JmxConnector exception, clientLogIdent:{} host:{} port:{}.", clientLogIdent, host, port, e);
LOGGER.error(
"method=close||clientLogIdent={}||jmxHost={}||jmxPort={}||msg=close jmx JmxConnector exception.",
clientLogIdent, jmxHost, jmxPort, e
);
}
}
@@ -159,7 +165,12 @@ public class JmxConnectorWrap {
if (jmxConnector != null) {
return true;
}
String jmxUrl = String.format("service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", host, port);
LOGGER.info(
"method=createJmxConnector||clientLogIdent={}||brokerStartupTime={}||jmxHost={}||jmxPort={}||jmxConfig={}||msg=start create jmx connector.",
clientLogIdent, brokerStartupTime, jmxHost, jmxPort, jmxConfig
);
String jmxUrl = String.format("service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", jmxHost, jmxPort);
try {
Map<String, Object> environment = new HashMap<String, Object>();
if (!ValidateUtils.isBlank(this.jmxConfig.getUsername()) && !ValidateUtils.isBlank(this.jmxConfig.getToken())) {
@@ -174,12 +185,21 @@ public class JmxConnectorWrap {
}
jmxConnector = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), environment);
LOGGER.info("JMX connect success, clientLogIdent:{} host:{} port:{}.", clientLogIdent, host, port);
LOGGER.info(
"method=createJmxConnector||clientLogIdent={}||jmxHost={}||jmxPort={}||msg=jmx connect success.",
clientLogIdent, jmxHost, jmxPort
);
return true;
} catch (MalformedURLException e) {
LOGGER.error("JMX url exception, clientLogIdent:{} host:{} port:{} jmxUrl:{}", clientLogIdent, host, port, jmxUrl, e);
LOGGER.error(
"method=createJmxConnector||clientLogIdent={}||jmxHost={}||jmxPort={}||jmxUrl={}||msg=jmx url exception.",
clientLogIdent, jmxHost, jmxPort, jmxUrl, e
);
} catch (Exception e) {
LOGGER.error("JMX connect exception, clientLogIdent:{} host:{} port:{}.", clientLogIdent, host, port, e);
LOGGER.error(
"method=createJmxConnector||clientLogIdent={}||jmxHost={}||jmxPort={}||msg=jmx connect exception.",
clientLogIdent, jmxHost, jmxPort, e
);
}
return false;
}

48
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/utils/kafka/KSPartialKafkaAdminClient.java
View File

@@ -78,6 +78,8 @@ import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.connect.runtime.distributed.ConnectProtocol;
import org.apache.kafka.connect.runtime.distributed.ExtendedWorkerState;
import org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol;
import org.slf4j.Logger;
import java.net.InetSocketAddress;
@@ -1338,21 +1340,11 @@ public class KSPartialKafkaAdminClient {
if (groupMember.memberAssignment().length > 0) {
final Assignment assignment = ConsumerProtocol.deserializeAssignment(ByteBuffer.wrap(groupMember.memberAssignment()));
memberBaseAssignment = new KSMemberConsumerAssignment(new HashSet<>(assignment.partitions()));
} else {
memberBaseAssignment = new KSMemberConsumerAssignment(new HashSet<>());
}
} else {
ConnectProtocol.Assignment assignment = null;
if (groupMember.memberAssignment().length > 0) {
assignment = ConnectProtocol.
deserializeAssignment(ByteBuffer.wrap(groupMember.memberAssignment()));
}
ConnectProtocol.WorkerState workerState = null;
if (groupMember.memberMetadata().length > 0) {
workerState = ConnectProtocol.
deserializeMetadata(ByteBuffer.wrap(groupMember.memberMetadata()));
}
memberBaseAssignment = new KSMemberConnectAssignment(assignment, workerState);
memberBaseAssignment = deserializeConnectGroupDataCompatibility(groupMember);
}
memberDescriptions.add(new KSMemberDescription(
@@ -1381,6 +1373,36 @@ public class KSPartialKafkaAdminClient {
};
}
private KSMemberBaseAssignment deserializeConnectGroupDataCompatibility(DescribedGroupMember groupMember) {
try {
// 高版本的反序列化方式
ExtendedWorkerState workerState = null;
if (groupMember.memberMetadata().length > 0) {
workerState = IncrementalCooperativeConnectProtocol.
deserializeMetadata(ByteBuffer.wrap(groupMember.memberMetadata()));
return new KSMemberConnectAssignment(workerState.assignment(), workerState);
}
} catch (Exception e) {
// ignore
}
// 低版本的反序列化方式
ConnectProtocol.Assignment assignment = null;
if (groupMember.memberAssignment().length > 0) {
assignment = ConnectProtocol.
deserializeAssignment(ByteBuffer.wrap(groupMember.memberAssignment()));
}
ConnectProtocol.WorkerState workerState = null;
if (groupMember.memberMetadata().length > 0) {
workerState = ConnectProtocol.
deserializeMetadata(ByteBuffer.wrap(groupMember.memberMetadata()));
}
return new KSMemberConnectAssignment(assignment, workerState);
}
private Set<AclOperation> validAclOperations(final int authorizedOperations) {
if (authorizedOperations == MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED) {