8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2026-01-03 11:28:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

v2.8.1_e初始化

Browse Source 
1、测试代码，开源用户尽量不要使用；
2、包含Kafka-HA的相关功能，在v2.8.0_e的基础上，补充按照clientId切换的功能；
3、基于v2.8.0_e拉的分支；
This commit is contained in:
zengqiao
2023-02-13 16:48:59 +08:00
parent e81c0f3040
commit b16a7b9bff
44 changed files with 1759 additions and 611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ha/HaResTypeEnum.java
View File

@@ -9,9 +9,13 @@ import lombok.Getter;
@Getter
public enum HaResTypeEnum {
CLUSTER(0, "Cluster"),
TOPIC(1, "Topic"),
KAFKA_USER(2, "KafkaUser"),
KAFKA_USER_AND_CLIENT(3, "KafkaUserAndClient"),
;
private final int code;
@@ -22,4 +26,4 @@ public enum HaResTypeEnum {
this.code = code;
this.msg = msg;
}
}
}

2
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/ConfigConstant.java
View File

@@ -33,6 +33,8 @@ public class ConfigConstant {
public static final String HA_SWITCH_JOB_TIMEOUT_UNIT_SEC_CONFIG_PREFIX = "HA_SWITCH_JOB_TIMEOUT_UNIT_SEC_CONFIG_CLUSTER";
public static final String HA_CONNECTION_ACTIVE_TIME_UNIT_MIN = "HA_CONNECTION_ACTIVE_TIME_UNIT_MIN";
private ConfigConstant() {
}
}

72
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/topic/TopicConnection.java
View File

@@ -1,9 +1,12 @@
package com.xiaojukeji.kafka.manager.common.entity.ao.topic;
import lombok.Data;
/**
* @author zengqiao
* @date 20/4/20
*/
@Data
public class TopicConnection {
private Long clusterId;
@@ -19,72 +22,9 @@ public class TopicConnection {
private String clientVersion;
public Long getClusterId() {
return clusterId;
}
private String clientId;
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
private Long realConnectTime;
public String getTopicName() {
return topicName;
}
public void setTopicName(String topicName) {
this.topicName = topicName;
}
public String getAppId() {
return appId;
}
public void setAppId(String appId) {
this.appId = appId;
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public String getClientType() {
return clientType;
}
public void setClientType(String clientType) {
this.clientType = clientType;
}
public String getClientVersion() {
return clientVersion;
}
public void setClientVersion(String clientVersion) {
this.clientVersion = clientVersion;
}
@Override
public String toString() {
return "TopicConnectionDTO{" +
"clusterId=" + clusterId +
", topicName='" + topicName + '\'' +
", appId='" + appId + '\'' +
", ip='" + ip + '\'' +
", hostname='" + hostname + '\'' +
", clientType='" + clientType + '\'' +
", clientVersion='" + clientVersion + '\'' +
'}';
}
private Long createTime;
}

8
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/ha/ASSwitchJobActionDTO.java
View File

@@ -15,12 +15,4 @@ public class ASSwitchJobActionDTO {
@NotBlank(message = "action不允许为空")
@ApiModelProperty(value = "动作, force")
private String action;
// @NotNull(message = "all不允许为NULL")
// @ApiModelProperty(value = "所有的Topic")
// private Boolean allJumpWaitInSync;
//
// @NotNull(message = "jumpWaitInSyncActiveTopicList不允许为NULL")
// @ApiModelProperty(value = "操作的Topic")
// private List<String> jumpWaitInSyncActiveTopicList;
}

9
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/ha/ASSwitchJobDTO.java
View File

@@ -4,6 +4,7 @@ import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.util.List;
@@ -27,5 +28,13 @@ public class ASSwitchJobDTO {
private Long standbyClusterPhyId;
@NotNull(message = "topicNameList不允许为NULL")
@ApiModelProperty(value="切换的Topic名称列表")
private List<String> topicNameList;
/**
* kafkaUser+Client列表
*/
@Valid
@ApiModelProperty(value="切换的KafkaUser&ClientId列表，Client可以为空串")
private List<KafkaUserAndClientDTO> kafkaUserAndClientIdList;
}

18
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/ha/KafkaUserAndClientDTO.java Normal file
View File

@@ -0,0 +1,18 @@
package com.xiaojukeji.kafka.manager.common.entity.dto.ha;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotBlank;
@Data
@ApiModel(description="KafkaUser和ClientId信息")
public class KafkaUserAndClientDTO {
@NotBlank(message = "kafkaUser不允许为空串")
@ApiModelProperty(value = "kafkaUser")
private String kafkaUser;
@ApiModelProperty(value = "clientId")
private String clientId;
}

4
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/topic/HaTopicRelationDTO.java
View File

@@ -32,6 +32,9 @@ public class HaTopicRelationDTO {
@ApiModelProperty(value = "需要关联|解绑的topic名称列表")
private List<String> topicNames;
@ApiModelProperty(value = "解绑是否保留备集群资源（topic,kafkaUser,group）")
private Boolean retainStandbyResource;
@Override
public String toString() {
return "HaTopicRelationDTO{" +
@@ -39,6 +42,7 @@ public class HaTopicRelationDTO {
", standbyClusterId=" + standbyClusterId +
", all=" + all +
", topicNames=" + topicNames +
", retainStandbyResource=" + retainStandbyResource +
'}';
}

7
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/rd/AppRelateTopicsDTO.java
View File

@@ -21,4 +21,11 @@ public class AppRelateTopicsDTO {
@NotNull(message = "filterTopicNameList不允许为NULL")
@ApiModelProperty(value="过滤的Topic列表")
private List<String> filterTopicNameList;
@ApiModelProperty(value="使用KafkaUser+Client维度的数据，默认是kafkaUser维度")
private Boolean useKafkaUserAndClientId;
@NotNull(message = "ha不允许为NULL")
@ApiModelProperty(value="查询是否高可用topic")
private Boolean ha;
}

87
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/gateway/TopicConnectionDO.java
View File

@@ -1,5 +1,7 @@
package com.xiaojukeji.kafka.manager.common.entity.pojo.gateway;
import lombok.Data;
import java.util.Date;
/**
@@ -7,6 +9,7 @@ import java.util.Date;
* @author zengqiao
* @date 20/7/6
*/
@Data
public class TopicConnectionDO {
private Long id;
@@ -22,87 +25,13 @@ public class TopicConnectionDO {
private String clientVersion;
private String clientId;
private Long realConnectTime;
private Date createTime;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public Long getClusterId() {
return clusterId;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
public String getTopicName() {
return topicName;
}
public void setTopicName(String topicName) {
this.topicName = topicName;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getAppId() {
return appId;
}
public void setAppId(String appId) {
this.appId = appId;
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getClientVersion() {
return clientVersion;
}
public void setClientVersion(String clientVersion) {
this.clientVersion = clientVersion;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
@Override
public String toString() {
return "TopicConnectionDO{" +
"id=" + id +
", clusterId=" + clusterId +
", topicName='" + topicName + '\'' +
", type='" + type + '\'' +
", appId='" + appId + '\'' +
", ip='" + ip + '\'' +
", clientVersion='" + clientVersion + '\'' +
", createTime=" + createTime +
'}';
}
public String uniqueKey() {
return appId + clusterId + topicName + type + ip;
return appId + clusterId + topicName + type + ip + clientId;
}
}

2
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/ha/HaASRelationDO.java
View File

@@ -1,6 +1,7 @@
package com.xiaojukeji.kafka.manager.common.entity.pojo.ha;
import com.baomidou.mybatisplus.annotation.TableName;
import com.xiaojukeji.kafka.manager.common.bizenum.ha.HaResTypeEnum;
import com.xiaojukeji.kafka.manager.common.entity.pojo.BaseDO;
import lombok.AllArgsConstructor;
import lombok.Data;
@@ -37,6 +38,7 @@ public class HaASRelationDO extends BaseDO {
/**
* 资源类型
* @see HaResTypeEnum
*/
private Integer resType;

28
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/pojo/ha/HaASSwitchJobDO.java
View File

@@ -1,10 +1,16 @@
package com.xiaojukeji.kafka.manager.common.entity.pojo.ha;
import com.baomidou.mybatisplus.annotation.TableName;
import com.xiaojukeji.kafka.manager.common.entity.dto.ha.KafkaUserAndClientDTO;
import com.xiaojukeji.kafka.manager.common.entity.pojo.BaseDO;
import com.xiaojukeji.kafka.manager.common.utils.ConvertUtil;
import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
/**
* HA-主备关系切换任务表
@@ -28,15 +34,35 @@ public class HaASSwitchJobDO extends BaseDO {
*/
private Integer jobStatus;
/**
* 类型，0：kafkaUser 1：kafkaUser+Client
*/
private Integer type;
/**
* 扩展数据
*/
private String extendData;
/**
* 操作人
*/
private String operator;
public HaASSwitchJobDO(Long activeClusterPhyId, Long standbyClusterPhyId, Integer jobStatus, String operator) {
public HaASSwitchJobDO(Long activeClusterPhyId, Long standbyClusterPhyId, Integer type, List<KafkaUserAndClientDTO> extendDataObj, Integer jobStatus, String operator) {
this.activeClusterPhyId = activeClusterPhyId;
this.standbyClusterPhyId = standbyClusterPhyId;
this.type = type;
this.extendData = ValidateUtils.isEmptyList(extendDataObj)? "": ConvertUtil.obj2Json(extendDataObj);
this.jobStatus = jobStatus;
this.operator = operator;
}
public List<KafkaUserAndClientDTO> getExtendRawData() {
if (ValidateUtils.isBlank(extendData)) {
return new ArrayList<>();
}
return ConvertUtil.str2ObjArrayByJson(extendData, KafkaUserAndClientDTO.class);
}
}

74
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/topic/TopicConnectionVO.java
View File

@@ -2,11 +2,13 @@ package com.xiaojukeji.kafka.manager.common.entity.vo.normal.topic;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author zhongyuankai,zengqiao
* @date 20/4/8
*/
@Data
@ApiModel(value = "Topic连接信息")
public class TopicConnectionVO {
@ApiModelProperty(value = "集群ID")
@@ -30,72 +32,12 @@ public class TopicConnectionVO {
@ApiModelProperty(value = "客户端版本")
private String clientVersion;
public Long getClusterId() {
return clusterId;
}
@ApiModelProperty(value = "客户端ID")
private String clientId;
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
@ApiModelProperty(value = "连接Broker时间")
private Long realConnectTime;
public String getTopicName() {
return topicName;
}
public void setTopicName(String topicName) {
this.topicName = topicName;
}
public String getAppId() {
return appId;
}
public void setAppId(String appId) {
this.appId = appId;
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public String getClientType() {
return clientType;
}
public void setClientType(String clientType) {
this.clientType = clientType;
}
public String getClientVersion() {
return clientVersion;
}
public void setClientVersion(String clientVersion) {
this.clientVersion = clientVersion;
}
@Override
public String toString() {
return "TopicConnectionVO{" +
"clusterId=" + clusterId +
", topicName='" + topicName + '\'' +
", appId='" + appId + '\'' +
", ip='" + ip + '\'' +
", hostname='" + hostname + '\'' +
", clientType='" + clientType + '\'' +
", clientVersion='" + clientVersion + '\'' +
'}';
}
@ApiModelProperty(value = "创建时间")
private Long createTime;
}

42
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/app/AppRelateTopicsVO.java
View File

@@ -3,7 +3,9 @@ package com.xiaojukeji.kafka.manager.common.entity.vo.rd.app;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
/**
@@ -11,6 +13,7 @@ import java.util.List;
* @date 20/5/4
*/
@Data
@NoArgsConstructor
@ApiModel(description="App关联Topic信息")
public class AppRelateTopicsVO {
@ApiModelProperty(value="物理集群ID")
@@ -19,6 +22,12 @@ public class AppRelateTopicsVO {
@ApiModelProperty(value="kafkaUser")
private String kafkaUser;
@ApiModelProperty(value="clientId")
private String clientId;
@ApiModelProperty(value="已建立HA的Client")
private List<String> haClientIdList;
@ApiModelProperty(value="选中的Topic列表")
private List<String> selectedTopicNameList;
@@ -27,4 +36,37 @@ public class AppRelateTopicsVO {
@ApiModelProperty(value="未建立HA的Topic列表")
private List<String> notHaTopicNameList;
public AppRelateTopicsVO(Long clusterPhyId, String kafkaUser, String clientId) {
this.clusterPhyId = clusterPhyId;
this.kafkaUser = kafkaUser;
this.clientId = clientId;
this.selectedTopicNameList = new ArrayList<>();
this.notSelectTopicNameList = new ArrayList<>();
this.notHaTopicNameList = new ArrayList<>();
}
public void addSelectedIfNotExist(String topicName) {
if (selectedTopicNameList.contains(topicName)) {
return;
}
selectedTopicNameList.add(topicName);
}
public void addNotSelectedIfNotExist(String topicName) {
if (notSelectTopicNameList.contains(topicName)) {
return;
}
notSelectTopicNameList.add(topicName);
}
public void addNotHaIfNotExist(String topicName) {
if (notHaTopicNameList.contains(topicName)) {
return;
}
notHaTopicNameList.add(topicName);
}
}

29
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/HAUtils.java Normal file
View File

@@ -0,0 +1,29 @@
package com.xiaojukeji.kafka.manager.common.utils;
public class HAUtils {
public static String mergeKafkaUserAndClient(String kafkaUser, String clientId) {
if (ValidateUtils.isBlank(clientId)) {
return kafkaUser;
}
return String.format("%s#%s", kafkaUser, clientId);
}
public static Tuple<String, String> splitKafkaUserAndClient(String kafkaUserAndClientId) {
if (ValidateUtils.isBlank(kafkaUserAndClientId)) {
return null;
}
int idx = kafkaUserAndClientId.indexOf('#');
if (idx == -1) {
return null;
} else if (idx == kafkaUserAndClientId.length() - 1) {
return new Tuple<>(kafkaUserAndClientId.substring(0, idx), "");
}
return new Tuple<>(kafkaUserAndClientId.substring(0, idx), kafkaUserAndClientId.substring(idx + 1));
}
private HAUtils() {
}
}

29
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java
View File

@@ -79,10 +79,27 @@ public class JsonUtils {
TopicConnectionDO connectionDO = new TopicConnectionDO();
String[] appIdDetailArray = appIdDetail.toString().split("#");
if (appIdDetailArray.length >= 3) {
connectionDO.setAppId(appIdDetailArray[0]);
connectionDO.setIp(appIdDetailArray[1]);
connectionDO.setClientVersion(appIdDetailArray[2]);
if (appIdDetailArray == null) {
appIdDetailArray = new String[0];
}
connectionDO.setAppId(parseTopicConnections(appIdDetailArray, 0));
connectionDO.setIp(parseTopicConnections(appIdDetailArray, 1));
connectionDO.setClientVersion(parseTopicConnections(appIdDetailArray, 2));
// 解析clientId
StringBuilder sb = new StringBuilder();
for (int i = 3; i < appIdDetailArray.length - 1; ++i) {
sb.append(parseTopicConnections(appIdDetailArray, i)).append("#");
}
connectionDO.setClientId(sb.substring(0, sb.length() - 1));
// 解析时间
Long receiveTime = ConvertUtil.string2Long(parseTopicConnections(appIdDetailArray, appIdDetailArray.length - 1));
if (receiveTime == null) {
connectionDO.setRealConnectTime(-1L);
} else {
connectionDO.setRealConnectTime(receiveTime);
}
connectionDO.setClusterId(clusterId);
@@ -95,4 +112,8 @@ public class JsonUtils {
}
return connectionDOList;
}
private static String parseTopicConnections(String[] appIdDetailArray, int idx) {
return (appIdDetailArray != null && appIdDetailArray.length >= idx + 1)? appIdDetailArray[idx]: "";
}
}

61
kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/Tuple.java Normal file
View File

@@ -0,0 +1,61 @@
package com.xiaojukeji.kafka.manager.common.utils;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
/**
* @Author: D10865
* @Description:
* @Date: Create on 2018/5/29 下午4:08
* @Modified By
*/
@JsonIgnoreProperties(value = { "hibernateLazyInitializer", "handler" })
@Data
public class Tuple<T, V> {
private T v1;
private V v2;
public Tuple(){}
public Tuple(T v1, V v2) {
this.v1 = v1;
this.v2 = v2;
}
public T v1() {
return v1;
}
public Tuple<T, V> setV1(T v1) {
this.v1 = v1;
return this;
}
public V v2() {
return v2;
}
public Tuple<T, V> setV2(V v2) {
this.v2 = v2;
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) {return true;}
if (o == null || getClass() != o.getClass()) {return false;}
Tuple<?, ?> tuple = (Tuple<?, ?>) o;
if (v1 != null ? !v1.equals(tuple.v1) : tuple.v1 != null) {return false;}
return v2 != null ? v2.equals(tuple.v2) : tuple.v2 == null;
}
@Override
public int hashCode() {
int result = v1 != null ? v1.hashCode() : 0;
result = 31 * result + (v2 != null ? v2.hashCode() : 0);
return result;
}
}