8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 03:42:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Connect相关代码

Browse Source
This commit is contained in:
zengqiao
2022-12-06 18:13:44 +08:00
committed by EricZeng
parent fb5964af84
commit e8652d5db5
169 changed files with 6472 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/group/impl/GroupManagerImpl.java
View File

@@ -8,9 +8,13 @@ import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetResetDT
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
import com.xiaojukeji.know.streaming.km.common.bean.dto.partition.PartitionOffsetDTO;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.Group;
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopic;
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopicMember;
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSGroupDescription;
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSMemberConsumerAssignment;
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSMemberDescription;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.offset.KSOffsetSpec;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
@@ -35,14 +39,13 @@ import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.PaginationMetricsUtil;
import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import com.xiaojukeji.know.streaming.km.core.service.group.GroupMetricService;
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka.GroupMetricVersionItems;
import com.xiaojukeji.know.streaming.km.persistence.es.dao.GroupMetricESDAO;
import org.apache.kafka.clients.admin.ConsumerGroupDescription;
import org.apache.kafka.clients.admin.MemberDescription;
import org.apache.kafka.common.ConsumerGroupState;
import org.apache.kafka.common.TopicPartition;
import org.springframework.beans.factory.annotation.Autowired;
@@ -51,6 +54,8 @@ import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
import static com.xiaojukeji.know.streaming.km.common.enums.group.GroupTypeEnum.CONNECT_CLUSTER_PROTOCOL_TYPE;
@Component
public class GroupManagerImpl implements GroupManager {
private static final ILog log = LogFactory.getLog(GroupManagerImpl.class);
@@ -70,6 +75,9 @@ public class GroupManagerImpl implements GroupManager {
@Autowired
private GroupMetricESDAO groupMetricESDAO;
@Autowired
private ClusterPhyService clusterPhyService;
@Override
public PaginationResult<GroupTopicOverviewVO> pagingGroupMembers(Long clusterPhyId,
String topicName,
@@ -140,6 +148,11 @@ public class GroupManagerImpl implements GroupManager {
String groupName,
List<String> latestMetricNames,
PaginationSortDTO dto) throws NotExistException, AdminOperateException {
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
if (clusterPhy == null) {
return PaginationResult.buildFailure(MsgConstant.getClusterPhyNotExist(clusterPhyId), dto);
}
// 获取消费组消费的TopicPartition列表
Map<TopicPartition, Long> consumedOffsetMap = groupService.getGroupOffsetFromKafka(clusterPhyId, groupName);
List<Integer> partitionList = consumedOffsetMap.keySet()
@@ -150,15 +163,20 @@ public class GroupManagerImpl implements GroupManager {
Collections.sort(partitionList);
// 获取消费组当前运行信息
ConsumerGroupDescription groupDescription = groupService.getGroupDescriptionFromKafka(clusterPhyId, groupName);
KSGroupDescription groupDescription = groupService.getGroupDescriptionFromKafka(clusterPhy, groupName);
// 转换存储格式
Map<TopicPartition, MemberDescription> tpMemberMap = new HashMap<>();
for (MemberDescription description: groupDescription.members()) {
for (TopicPartition tp: description.assignment().topicPartitions()) {
Map<TopicPartition, KSMemberDescription> tpMemberMap = new HashMap<>();
//如果不是connect集群
if (!groupDescription.protocolType().equals(CONNECT_CLUSTER_PROTOCOL_TYPE)) {
for (KSMemberDescription description : groupDescription.members()) {
KSMemberConsumerAssignment assignment = (KSMemberConsumerAssignment) description.assignment();
for (TopicPartition tp : assignment.topicPartitions()) {
tpMemberMap.put(tp, description);
}
}
}
// 获取指标
PaginationResult<GroupMetrics> metricsResult = this.pagingGroupTopicPartitionMetrics(clusterPhyId, groupName, topicName, partitionList, latestMetricNames, dto);
@@ -173,11 +191,11 @@ public class GroupManagerImpl implements GroupManager {
vo.setTopicName(topicName);
vo.setPartitionId(groupMetrics.getPartitionId());
MemberDescription memberDescription = tpMemberMap.get(new TopicPartition(topicName, groupMetrics.getPartitionId()));
if (memberDescription != null) {
vo.setMemberId(memberDescription.consumerId());
vo.setHost(memberDescription.host());
vo.setClientId(memberDescription.clientId());
KSMemberDescription ksMemberDescription = tpMemberMap.get(new TopicPartition(topicName, groupMetrics.getPartitionId()));
if (ksMemberDescription != null) {
vo.setMemberId(ksMemberDescription.consumerId());
vo.setHost(ksMemberDescription.host());
vo.setClientId(ksMemberDescription.clientId());
}
vo.setLatestMetrics(groupMetrics);
@@ -203,7 +221,12 @@ public class GroupManagerImpl implements GroupManager {
return rv;
}
ConsumerGroupDescription description = groupService.getGroupDescriptionFromKafka(dto.getClusterId(), dto.getGroupName());
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(dto.getClusterId());
if (clusterPhy == null) {
return Result.buildFromRSAndMsg(ResultStatus.CLUSTER_NOT_EXIST, MsgConstant.getClusterPhyNotExist(dto.getClusterId()));
}
KSGroupDescription description = groupService.getGroupDescriptionFromKafka(clusterPhy, dto.getGroupName());
if (ConsumerGroupState.DEAD.equals(description.state()) && !dto.isCreateIfNotExist()) {
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, "group不存在, 重置失败");
}
@@ -345,32 +368,4 @@ public class GroupManagerImpl implements GroupManager {
dto
);
}
private List<GroupTopicOverviewVO> convert2GroupTopicOverviewVOList(String groupName, String state, List<GroupTopicMember> groupTopicList, List<GroupMetrics> metricsList) {
if (metricsList == null) {
metricsList = new ArrayList<>();
}
// <TopicName, GroupMetrics>
Map<String, GroupMetrics> metricsMap = new HashMap<>();
for (GroupMetrics metrics : metricsList) {
if (!groupName.equals(metrics.getGroup())) continue;
metricsMap.put(metrics.getTopic(), metrics);
}
List<GroupTopicOverviewVO> voList = new ArrayList<>();
for (GroupTopicMember po : groupTopicList) {
GroupTopicOverviewVO vo = ConvertUtil.obj2Obj(po, GroupTopicOverviewVO.class);
vo.setGroupName(groupName);
vo.setState(state);
GroupMetrics metrics = metricsMap.get(po.getTopicName());
if (metrics != null) {
vo.setMaxLag(ConvertUtil.Float2Long(metrics.getMetrics().get(GroupMetricVersionItems.GROUP_METRIC_LAG)));
}
voList.add(vo);
}
return voList;
}
}

4
km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/topic/impl/TopicConfigManagerImpl.java
View File

@@ -16,7 +16,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerConfigService;
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
import com.xiaojukeji.know.streaming.km.core.service.version.BaseKafkaVersionControlService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
@Component
public class TopicConfigManagerImpl extends BaseVersionControlService implements TopicConfigManager {
public class TopicConfigManagerImpl extends BaseKafkaVersionControlService implements TopicConfigManager {
private static final ILog log = LogFactory.getLog(TopicConfigManagerImpl.class);
private static final String GET_DEFAULT_TOPIC_CONFIG = "getDefaultTopicConfig";

4
km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/VersionControlManager.java
View File

@@ -20,7 +20,7 @@ public interface VersionControlManager {
* 获取当前ks所有支持的kafka版本
* @return
*/
Result<Map<String, Long>> listAllVersions();
Result<Map<String, Long>> listAllKafkaVersions();
/**
* 获取全部集群 clusterId 中类型为 type 的指标，不论支持不支持
@@ -28,7 +28,7 @@ public interface VersionControlManager {
* @param type
* @return
*/
Result<List<VersionItemVO>> listClusterVersionControlItem(Long clusterId, Integer type);
Result<List<VersionItemVO>> listKafkaClusterVersionControlItem(Long clusterId, Integer type);
/**
* 获取当前用户设置的用于展示的指标配置

20
km-biz/src/main/java/com/xiaojukeji/know/streaming/km/biz/version/impl/VersionControlManagerImpl.java
View File

@@ -17,6 +17,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.version.VersionItemVO;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.VersionUtil;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -92,6 +93,9 @@ public class VersionControlManagerImpl implements VersionControlManager {
defaultMetrics.add(new UserMetricConfig(METRIC_BROKER.getCode(), BROKER_METRIC_BYTES_OUT, true));
}
@Autowired
private ClusterPhyService clusterPhyService;
@Autowired
private VersionControlService versionControlService;
@@ -107,7 +111,13 @@ public class VersionControlManagerImpl implements VersionControlManager {
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_BROKER.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_PARTITION.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_REPLICATION.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_ZOOKEEPER.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CONNECT_CLUSTER.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CONNECT_CONNECTOR.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(METRIC_CONNECT_MIRROR_MAKER.getCode()), VersionItemVO.class));
allVersionItemVO.addAll(ConvertUtil.list2List(versionControlService.listVersionControlItem(WEB_OP.getCode()), VersionItemVO.class));
Map<String, VersionItemVO> map = allVersionItemVO.stream().collect(
@@ -121,18 +131,20 @@ public class VersionControlManagerImpl implements VersionControlManager {
}
@Override
public Result<Map<String, Long>> listAllVersions() {
public Result<Map<String, Long>> listAllKafkaVersions() {
return Result.buildSuc(VersionEnum.allVersionsWithOutMax());
}
@Override
public Result<List<VersionItemVO>> listClusterVersionControlItem(Long clusterId, Integer type) {
public Result<List<VersionItemVO>> listKafkaClusterVersionControlItem(Long clusterId, Integer type) {
List<VersionControlItem> allItem = versionControlService.listVersionControlItem(type);
List<VersionItemVO> versionItemVOS = new ArrayList<>();
String versionStr = clusterPhyService.getVersionFromCacheFirst(clusterId);
for (VersionControlItem item : allItem){
VersionItemVO itemVO = ConvertUtil.obj2Obj(item, VersionItemVO.class);
boolean support = versionControlService.isClusterSupport(clusterId, item);
boolean support = versionControlService.isClusterSupport(versionStr, item);
itemVO.setSupport(support);
itemVO.setDesc(itemSupportDesc(item, support));
@@ -145,7 +157,7 @@ public class VersionControlManagerImpl implements VersionControlManager {
@Override
public Result<List<UserMetricConfigVO>> listUserMetricItem(Long clusterId, Integer type, String operator) {
Result<List<VersionItemVO>> ret = listClusterVersionControlItem(clusterId, type);
Result<List<VersionItemVO>> ret = listKafkaClusterVersionControlItem(clusterId, type);
if(null == ret || ret.failed()){
return Result.buildFail();
}

32
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/AbstractMetricCollector.java
View File

@@ -1,52 +1,26 @@
package com.xiaojukeji.know.streaming.km.collector.metric;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.service.CollectThreadPoolService;
import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BaseMetricEvent;
import com.xiaojukeji.know.streaming.km.common.component.SpringTool;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
/**
* @author didi
*/
public abstract class AbstractMetricCollector<T> {
protected static final ILog LOGGER = LogFactory.getLog(AbstractMetricCollector.class);
protected static final ILog METRIC_COLLECTED_LOGGER = LoggerUtil.getMetricCollectedLogger();
public abstract List<T> collectKafkaMetrics(ClusterPhy clusterPhy);
public abstract class AbstractMetricCollector<M, C> {
public abstract String getClusterVersion(C c);
public abstract VersionItemTypeEnum collectorType();
@Autowired
private CollectThreadPoolService collectThreadPoolService;
public void collectMetrics(ClusterPhy clusterPhy) {
long startTime = System.currentTimeMillis();
// 采集指标
List<T> metricsList = this.collectKafkaMetrics(clusterPhy);
// 输出耗时信息
LOGGER.info(
"metricType={}||clusterPhyId={}||costTimeUnitMs={}",
this.collectorType().getMessage(), clusterPhy.getId(), System.currentTimeMillis() - startTime
);
// 输出采集到的指标信息
METRIC_COLLECTED_LOGGER.debug("metricType={}||clusterPhyId={}||metrics={}!",
this.collectorType().getMessage(), clusterPhy.getId(), ConvertUtil.obj2Json(metricsList)
);
}
public abstract void collectMetrics(C c);
protected FutureWaitUtil<Void> getFutureUtilByClusterPhyId(Long clusterPhyId) {
return collectThreadPoolService.selectSuitableFutureUtil(clusterPhyId * 1000L + this.collectorType().getCode());

50
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/connect/AbstractConnectMetricCollector.java Normal file
View File

@@ -0,0 +1,50 @@
package com.xiaojukeji.know.streaming.km.collector.metric.connect;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil;
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterService;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
/**
* @author didi
*/
public abstract class AbstractConnectMetricCollector<M> extends AbstractMetricCollector<M, ConnectCluster> {
private static final ILog LOGGER = LogFactory.getLog(AbstractConnectMetricCollector.class);
protected static final ILog METRIC_COLLECTED_LOGGER = LoggerUtil.getMetricCollectedLogger();
@Autowired
private ConnectClusterService connectClusterService;
public abstract List<M> collectConnectMetrics(ConnectCluster connectCluster);
@Override
public String getClusterVersion(ConnectCluster connectCluster){
return connectClusterService.getClusterVersion(connectCluster.getId());
}
@Override
public void collectMetrics(ConnectCluster connectCluster) {
long startTime = System.currentTimeMillis();
// 采集指标
List<M> metricsList = this.collectConnectMetrics(connectCluster);
// 输出耗时信息
LOGGER.info(
"metricType={}||connectClusterId={}||costTimeUnitMs={}",
this.collectorType().getMessage(), connectCluster.getId(), System.currentTimeMillis() - startTime
);
// 输出采集到的指标信息
METRIC_COLLECTED_LOGGER.debug("metricType={}||connectClusterId={}||metrics={}!",
this.collectorType().getMessage(), connectCluster.getId(), ConvertUtil.obj2Json(metricsList)
);
}
}

83
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/connect/ConnectClusterMetricCollector.java Normal file
View File

@@ -0,0 +1,83 @@
package com.xiaojukeji.know.streaming.km.collector.metric.connect;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectClusterMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectClusterMetricEvent;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterMetricService;
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.List;
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CONNECT_CLUSTER;
/**
* @author didi
*/
@Component
public class ConnectClusterMetricCollector extends AbstractConnectMetricCollector<ConnectClusterMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(ConnectClusterMetricCollector.class);
@Autowired
private VersionControlService versionControlService;
@Autowired
private ConnectClusterMetricService connectClusterMetricService;
@Override
public List<ConnectClusterMetrics> collectConnectMetrics(ConnectCluster connectCluster) {
Long startTime = System.currentTimeMillis();
Long clusterPhyId = connectCluster.getKafkaClusterPhyId();
Long connectClusterId = connectCluster.getId();
ConnectClusterMetrics metrics = new ConnectClusterMetrics(clusterPhyId, connectClusterId);
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
List<VersionControlItem> items = versionControlService.listVersionControlItem(getClusterVersion(connectCluster), collectorType().getCode());
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(connectClusterId);
for (VersionControlItem item : items) {
future.runnableTask(
String.format("class=ConnectClusterMetricCollector||connectClusterId=%d||metricName=%s", connectClusterId, item.getName()),
30000,
() -> {
try {
Result<ConnectClusterMetrics> ret = connectClusterMetricService.collectConnectClusterMetricsFromKafka(connectClusterId, item.getName());
if (null == ret || !ret.hasData()) {
return null;
}
metrics.putMetric(ret.getData().getMetrics());
} catch (Exception e) {
LOGGER.error(
"method=collectConnectMetrics||connectClusterId={}||metricName={}||errMsg=exception!",
connectClusterId, item.getName(), e
);
}
return null;
}
);
}
future.waitExecute(30000);
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
this.publishMetric(new ConnectClusterMetricEvent(this, Collections.singletonList(metrics)));
return Collections.singletonList(metrics);
}
@Override
public VersionItemTypeEnum collectorType() {
return METRIC_CONNECT_CLUSTER;
}
}

102
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/connect/ConnectConnectorMetricCollector.java Normal file
View File

@@ -0,0 +1,102 @@
package com.xiaojukeji.know.streaming.km.collector.metric.connect;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectorMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectorMetricEvent;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.enums.connect.ConnectorTypeEnum;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorMetricService;
import com.xiaojukeji.know.streaming.km.core.service.connect.connector.ConnectorService;
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CONNECT_CONNECTOR;
/**
* @author didi
*/
@Component
public class ConnectConnectorMetricCollector extends AbstractConnectMetricCollector<ConnectorMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(ConnectConnectorMetricCollector.class);
@Autowired
private VersionControlService versionControlService;
@Autowired
private ConnectorService connectorService;
@Autowired
private ConnectorMetricService connectorMetricService;
@Override
public List<ConnectorMetrics> collectConnectMetrics(ConnectCluster connectCluster) {
Long clusterPhyId = connectCluster.getKafkaClusterPhyId();
Long connectClusterId = connectCluster.getId();
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(connectCluster), collectorType().getCode());
Result<List<String>> connectorList = connectorService.listConnectorsFromCluster(connectClusterId);
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(connectClusterId);
List<ConnectorMetrics> metricsList = new ArrayList<>();
for (String connectorName : connectorList.getData()) {
ConnectorMetrics metrics = new ConnectorMetrics(connectClusterId, connectorName);
metrics.setClusterPhyId(clusterPhyId);
metricsList.add(metrics);
future.runnableTask(
String.format("class=ConnectConnectorMetricCollector||connectClusterId=%d||connectorName=%s", connectClusterId, connectorName),
30000,
() -> collectMetrics(connectClusterId, connectorName, metrics, items)
);
}
future.waitResult(30000);
this.publishMetric(new ConnectorMetricEvent(this, metricsList));
return metricsList;
}
@Override
public VersionItemTypeEnum collectorType() {
return METRIC_CONNECT_CONNECTOR;
}
/**************************************************** private method ****************************************************/
private void collectMetrics(Long connectClusterId, String connectorName, ConnectorMetrics metrics, List<VersionControlItem> items) {
long startTime = System.currentTimeMillis();
ConnectorTypeEnum connectorType = connectorService.getConnectorType(connectClusterId, connectorName);
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);
for (VersionControlItem v : items) {
try {
Result<ConnectorMetrics> ret = connectorMetricService.collectConnectClusterMetricsFromKafka(connectClusterId, connectorName, v.getName(), connectorType);
if (null == ret || ret.failed() || null == ret.getData()) {
continue;
}
metrics.putMetric(ret.getData().getMetrics());
} catch (Exception e) {
LOGGER.error(
"method=collectMetrics||connectClusterId={}||connectorName={}||metric={}||errMsg=exception!",
connectClusterId, connectorName, v.getName(), e
);
}
}
// 记录采集性能
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, (System.currentTimeMillis() - startTime) / 1000.0f);
}
}

50
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/AbstractKafkaMetricCollector.java Normal file
View File

@@ -0,0 +1,50 @@
package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
/**
* @author didi
*/
public abstract class AbstractKafkaMetricCollector<M> extends AbstractMetricCollector<M, ClusterPhy> {
private static final ILog LOGGER = LogFactory.getLog(AbstractMetricCollector.class);
protected static final ILog METRIC_COLLECTED_LOGGER = LoggerUtil.getMetricCollectedLogger();
@Autowired
private ClusterPhyService clusterPhyService;
public abstract List<M> collectKafkaMetrics(ClusterPhy clusterPhy);
@Override
public String getClusterVersion(ClusterPhy clusterPhy){
return clusterPhyService.getVersionFromCacheFirst(clusterPhy.getId());
}
@Override
public void collectMetrics(ClusterPhy clusterPhy) {
long startTime = System.currentTimeMillis();
// 采集指标
List<M> metricsList = this.collectKafkaMetrics(clusterPhy);
// 输出耗时信息
LOGGER.info(
"metricType={}||clusterPhyId={}||costTimeUnitMs={}",
this.collectorType().getMessage(), clusterPhy.getId(), System.currentTimeMillis() - startTime
);
// 输出采集到的指标信息
METRIC_COLLECTED_LOGGER.debug("metricType={}||clusterPhyId={}||metrics={}!",
this.collectorType().getMessage(), clusterPhy.getId(), ConvertUtil.obj2Json(metricsList)
);
}
}

5
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/BrokerMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
@@ -27,7 +26,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class BrokerMetricCollector extends AbstractMetricCollector<BrokerMetrics> {
public class BrokerMetricCollector extends AbstractKafkaMetricCollector<BrokerMetrics> {
private static final ILog LOGGER = LogFactory.getLog(BrokerMetricCollector.class);
@Autowired
@@ -44,7 +43,7 @@ public class BrokerMetricCollector extends AbstractMetricCollector<BrokerMetrics
Long clusterPhyId = clusterPhy.getId();
List<Broker> brokers = brokerService.listAliveBrokersFromDB(clusterPhy.getId());
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);

5
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/ClusterMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
@@ -25,7 +24,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class ClusterMetricCollector extends AbstractMetricCollector<ClusterMetrics> {
public class ClusterMetricCollector extends AbstractKafkaMetricCollector<ClusterMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(ClusterMetricCollector.class);
@Autowired
@@ -38,7 +37,7 @@ public class ClusterMetricCollector extends AbstractMetricCollector<ClusterMetri
public List<ClusterMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
Long startTime = System.currentTimeMillis();
Long clusterPhyId = clusterPhy.getId();
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
ClusterMetrics metrics = new ClusterMetrics(clusterPhyId, clusterPhy.getKafkaVersion());
metrics.putMetric(Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME, Constant.COLLECT_METRICS_ERROR_COST_TIME);

7
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/GroupMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
@@ -28,7 +27,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class GroupMetricCollector extends AbstractMetricCollector<GroupMetrics> {
public class GroupMetricCollector extends AbstractKafkaMetricCollector<GroupMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(GroupMetricCollector.class);
@Autowired
@@ -46,7 +45,7 @@ public class GroupMetricCollector extends AbstractMetricCollector<GroupMetrics>
List<String> groupNameList = new ArrayList<>();
try {
groupNameList = groupService.listGroupsFromKafka(clusterPhyId);
groupNameList = groupService.listGroupsFromKafka(clusterPhy);
} catch (Exception e) {
LOGGER.error("method=collectKafkaMetrics||clusterPhyId={}||msg=exception!", clusterPhyId, e);
}
@@ -55,7 +54,7 @@ public class GroupMetricCollector extends AbstractMetricCollector<GroupMetrics>
return Collections.emptyList();
}
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);

5
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/PartitionMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
@@ -26,7 +25,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class PartitionMetricCollector extends AbstractMetricCollector<PartitionMetrics> {
public class PartitionMetricCollector extends AbstractKafkaMetricCollector<PartitionMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(PartitionMetricCollector.class);
@Autowired
@@ -42,7 +41,7 @@ public class PartitionMetricCollector extends AbstractMetricCollector<PartitionM
public List<PartitionMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
Long clusterPhyId = clusterPhy.getId();
List<Topic> topicList = topicService.listTopicsFromCacheFirst(clusterPhyId);
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);

5
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/ReplicaMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
@@ -27,7 +26,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class ReplicaMetricCollector extends AbstractMetricCollector<ReplicationMetrics> {
public class ReplicaMetricCollector extends AbstractKafkaMetricCollector<ReplicationMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(ReplicaMetricCollector.class);
@Autowired
@@ -42,8 +41,8 @@ public class ReplicaMetricCollector extends AbstractMetricCollector<ReplicationM
@Override
public List<ReplicationMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
Long clusterPhyId = clusterPhy.getId();
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<Partition> partitions = partitionService.listPartitionFromCacheFirst(clusterPhyId);
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);

5
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/TopicMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
@@ -30,7 +29,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class TopicMetricCollector extends AbstractMetricCollector<TopicMetrics> {
public class TopicMetricCollector extends AbstractKafkaMetricCollector<TopicMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(TopicMetricCollector.class);
@Autowired
@@ -48,7 +47,7 @@ public class TopicMetricCollector extends AbstractMetricCollector<TopicMetrics>
public List<TopicMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
Long clusterPhyId = clusterPhy.getId();
List<Topic> topics = topicService.listTopicsFromCacheFirst(clusterPhyId);
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
FutureWaitUtil<Void> future = this.getFutureUtilByClusterPhyId(clusterPhyId);

5
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/metric/kafka/ZookeeperMetricCollector.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.collector.metric.kafka;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.metric.AbstractMetricCollector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.ZKConfig;
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
@@ -34,7 +33,7 @@ import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemT
* @author didi
*/
@Component
public class ZookeeperMetricCollector extends AbstractMetricCollector<ZookeeperMetrics> {
public class ZookeeperMetricCollector extends AbstractKafkaMetricCollector<ZookeeperMetrics> {
protected static final ILog LOGGER = LogFactory.getLog(ZookeeperMetricCollector.class);
@Autowired
@@ -53,7 +52,7 @@ public class ZookeeperMetricCollector extends AbstractMetricCollector<ZookeeperM
public List<ZookeeperMetrics> collectKafkaMetrics(ClusterPhy clusterPhy) {
Long startTime = System.currentTimeMillis();
Long clusterPhyId = clusterPhy.getId();
List<VersionControlItem> items = versionControlService.listVersionControlItem(clusterPhyId, collectorType().getCode());
List<VersionControlItem> items = versionControlService.listVersionControlItem(this.getClusterVersion(clusterPhy), collectorType().getCode());
List<ZookeeperInfo> aliveZKList = zookeeperService.listFromDBByCluster(clusterPhyId)
.stream()
.filter(elem -> Constant.ALIVE.equals(elem.getStatus()))

33
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/sink/connect/ConnectClusterMetricESSender.java Normal file
View File

@@ -0,0 +1,33 @@
package com.xiaojukeji.know.streaming.km.collector.sink.connect;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectClusterMetricEvent;
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectClusterMetricPO;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CONNECT_CLUSTER_INDEX;
/**
* @author wyb
* @date 2022/11/7
*/
@Component
public class ConnectClusterMetricESSender extends AbstractMetricESSender implements ApplicationListener<ConnectClusterMetricEvent> {
protected static final ILog LOGGER = LogFactory.getLog(ConnectClusterMetricESSender.class);
@PostConstruct
public void init(){
LOGGER.info("class=ConnectClusterMetricESSender||method=init||msg=init finished");
}
@Override
public void onApplicationEvent(ConnectClusterMetricEvent event) {
send2es(CONNECT_CLUSTER_INDEX, ConvertUtil.list2List(event.getConnectClusterMetrics(), ConnectClusterMetricPO.class));
}
}

33
km-collector/src/main/java/com/xiaojukeji/know/streaming/km/collector/sink/connect/ConnectorMetricESSender.java Normal file
View File

@@ -0,0 +1,33 @@
package com.xiaojukeji.know.streaming.km.collector.sink.connect;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.collector.sink.AbstractMetricESSender;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect.ConnectorMetricEvent;
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectorMetricPO;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CONNECT_CONNECTOR_INDEX;
/**
* @author wyb
* @date 2022/11/7
*/
@Component
public class ConnectorMetricESSender extends AbstractMetricESSender implements ApplicationListener<ConnectorMetricEvent> {
protected static final ILog LOGGER = LogFactory.getLog(ConnectorMetricESSender.class);
@PostConstruct
public void init(){
LOGGER.info("class=ConnectorMetricESSender||method=init||msg=init finished");
}
@Override
public void onApplicationEvent(ConnectorMetricEvent event) {
send2es(CONNECT_CONNECTOR_INDEX, ConvertUtil.list2List(event.getConnectorMetricsList(), ConnectorMetricPO.class));
}
}

28
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/cluster/ClusterConnectorsOverviewDTO.java Normal file
View File

@@ -0,0 +1,28 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.cluster;
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.List;
/**
* @author zengqiao
* @date 22/02/24
*/
@Data
public class ClusterConnectorsOverviewDTO extends PaginationSortDTO {
@NotNull(message = "latestMetricNames不允许为空")
@ApiModelProperty("需要指标点的信息")
private List<String> latestMetricNames;
@NotNull(message = "metricLines不允许为空")
@ApiModelProperty("需要指标曲线的信息")
private MetricDTO metricLines;
@ApiModelProperty("需要排序的指标名称列表,比较第一个不为空的metric")
private List<String> sortMetricNameList;
}

32
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/ClusterConnectorDTO.java Normal file
View File

@@ -0,0 +1,32 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect;
import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@NoArgsConstructor
@ApiModel(description = "集群Connector")
public class ClusterConnectorDTO extends BaseDTO {
@NotNull(message = "connectClusterId不允许为空")
@ApiModelProperty(value = "Connector集群ID", example = "1")
private Long connectClusterId;
@NotBlank(message = "name不允许为空串")
@ApiModelProperty(value = "Connector名称", example = "know-streaming-connector")
private String connectorName;
public ClusterConnectorDTO(Long connectClusterId, String connectorName) {
this.connectClusterId = connectClusterId;
this.connectorName = connectorName;
}
}

29
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/cluster/ConnectClusterDTO.java Normal file
View File

@@ -0,0 +1,29 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.cluster;
import com.xiaojukeji.know.streaming.km.common.bean.dto.BaseDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@ApiModel(description = "集群Connector")
public class ConnectClusterDTO extends BaseDTO {
@ApiModelProperty(value = "Connect集群ID", example = "1")
private Long id;
@ApiModelProperty(value = "Connect集群名称", example = "know-streaming")
private String name;
@ApiModelProperty(value = "Connect集群URL", example = "http://127.0.0.1:8080")
private String clusterUrl;
@ApiModelProperty(value = "Connect集群版本", example = "2.5.1")
private String version;
@ApiModelProperty(value = "JMX配置", example = "")
private String jmxProperties;
}

20
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/connector/ConnectorActionDTO.java Normal file
View File

@@ -0,0 +1,20 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotBlank;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@ApiModel(description = "操作Connector")
public class ConnectorActionDTO extends ClusterConnectorDTO {
@NotBlank(message = "action不允许为空串")
@ApiModelProperty(value = "Connector名称", example = "stop|restart|resume")
private String action;
}

21
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/connector/ConnectorConfigModifyDTO.java Normal file
View File

@@ -0,0 +1,21 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.Properties;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@ApiModel(description = "修改Connector配置")
public class ConnectorConfigModifyDTO extends ClusterConnectorDTO {
@NotNull(message = "configs不允许为空")
@ApiModelProperty(value = "配置", example = "")
private Properties configs;
}

21
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/connector/ConnectorCreateDTO.java Normal file
View File

@@ -0,0 +1,21 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotNull;
import java.util.Properties;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@ApiModel(description = "创建Connector")
public class ConnectorCreateDTO extends ClusterConnectorDTO {
@NotNull(message = "configs不允许为空")
@ApiModelProperty(value = "配置", example = "")
private Properties configs;
}

14
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/connector/ConnectorDeleteDTO.java Normal file
View File

@@ -0,0 +1,14 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
import io.swagger.annotations.ApiModel;
import lombok.Data;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@ApiModel(description = "删除Connector")
public class ConnectorDeleteDTO extends ClusterConnectorDTO {
}

20
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/connect/task/TaskActionDTO.java Normal file
View File

@@ -0,0 +1,20 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.connect.task;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.connector.ConnectorActionDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotNull;
/**
* @author zengqiao
* @date 2022-10-17
*/
@Data
@ApiModel(description = "操作Task")
public class TaskActionDTO extends ConnectorActionDTO {
@NotNull(message = "taskId不允许为NULL")
@ApiModelProperty(value = "taskId", example = "123")
private Long taskId;
}

22
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/connect/MetricsConnectClustersDTO.java Normal file
View File

@@ -0,0 +1,22 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.connect;
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* @author didi
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ApiModel(description = "Connect集群指标查询信息")
public class MetricsConnectClustersDTO extends MetricDTO {
@ApiModelProperty("Connect集群ID")
private List<Long> connectClusterIdList;
}

23
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/dto/metrices/connect/MetricsConnectorsDTO.java Normal file
View File

@@ -0,0 +1,23 @@
package com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.connect;
import com.xiaojukeji.know.streaming.km.common.bean.dto.connect.ClusterConnectorDTO;
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* @author didi
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ApiModel(description = "Connector指标查询信息")
public class MetricsConnectorsDTO extends MetricDTO {
@ApiModelProperty("Connector列表")
private List<ClusterConnectorDTO> connectorNameList;
}

1
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/broker/Broker.java
View File

@@ -3,7 +3,6 @@ package com.xiaojukeji.know.streaming.km.common.bean.entity.broker;
import com.alibaba.fastjson.TypeReference;
import com.xiaojukeji.know.streaming.km.common.bean.entity.common.IpPortData;
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig;
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import lombok.AllArgsConstructor;

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/config/metric/UserMetricConfig.java
View File

@@ -1,7 +1,5 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.config.metric;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

61
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/ConnectCluster.java Normal file
View File

@@ -0,0 +1,61 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.EntityIdInterface;
import lombok.Data;
import java.io.Serializable;
@Data
public class ConnectCluster implements Serializable, Comparable<ConnectCluster>, EntityIdInterface {
/**
* 集群ID
*/
private Long id;
/**
* 集群名字
*/
private String name;
/**
* 集群使用的消费组
*/
private String groupName;
/**
* 集群使用的消费组状态，也表示集群状态
* @see com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum
*/
private Integer state;
/**
* worker中显示的leader url信息
*/
private String memberLeaderUrl;
/**
* 版本信息
*/
private String version;
/**
* jmx配置
* @see com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig
*/
private String jmxProperties;
/**
* Kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* 集群地址
*/
private String clusterUrl;
@Override
public int compareTo(ConnectCluster connectCluster) {
return this.id.compareTo(connectCluster.getId());
}
}

38
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/ConnectClusterMetadata.java Normal file
View File

@@ -0,0 +1,38 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect;
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
@Data
@NoArgsConstructor
public class ConnectClusterMetadata implements Serializable {
/**
* Kafka集群名字
*/
private Long kafkaClusterPhyId;
/**
* 集群使用的消费组
*/
private String groupName;
/**
* 集群使用的消费组状态，也表示集群状态
*/
private GroupStateEnum state;
/**
* worker中显示的leader url信息
*/
private String memberLeaderUrl;
public ConnectClusterMetadata(Long kafkaClusterPhyId, String groupName, GroupStateEnum state, String memberLeaderUrl) {
this.kafkaClusterPhyId = kafkaClusterPhyId;
this.groupName = groupName;
this.state = state;
this.memberLeaderUrl = memberLeaderUrl;
}
}

87
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/ConnectWorker.java Normal file
View File

@@ -0,0 +1,87 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.net.URI;
@Data
@NoArgsConstructor
public class ConnectWorker implements Serializable {
protected static final ILog LOGGER = LogFactory.getLog(ConnectWorker.class);
/**
* Kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* 集群ID
*/
private Long connectClusterId;
/**
* 成员ID
*/
private String memberId;
/**
* 主机
*/
private String host;
/**
* Jmx端口
*/
private Integer jmxPort;
/**
* URL
*/
private String url;
/**
* leader的URL
*/
private String leaderUrl;
/**
* 1：是leader，0：不是leader
*/
private Integer leader;
/**
* worker地址
*/
private String workerId;
public ConnectWorker(Long kafkaClusterPhyId,
Long connectClusterId,
String memberId,
String host,
Integer jmxPort,
String url,
String leaderUrl,
Integer leader) {
this.kafkaClusterPhyId = kafkaClusterPhyId;
this.connectClusterId = connectClusterId;
this.memberId = memberId;
this.host = host;
this.jmxPort = jmxPort;
this.url = url;
this.leaderUrl = leaderUrl;
this.leader = leader;
String workerId = CommonUtils.getWorkerId(url);
if (workerId == null) {
workerId = memberId;
LOGGER.error("class=ConnectWorker||connectClusterId={}||memberId={}||url={}||msg=analysis url fail"
, connectClusterId, memberId, url);
}
this.workerId = workerId;
}
}

58
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/WorkerConnector.java Normal file
View File

@@ -0,0 +1,58 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
@Data
@NoArgsConstructor
public class WorkerConnector implements Serializable {
/**
* connect集群ID
*/
private Long connectClusterId;
/**
* kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* connector名称
*/
private String connectorName;
private String workerMemberId;
/**
* 任务状态
*/
private String state;
/**
* 任务ID
*/
private Integer taskId;
/**
* worker信息
*/
private String workerId;
/**
* 错误原因
*/
private String trace;
public WorkerConnector(Long kafkaClusterPhyId, Long connectClusterId, String connectorName, String workerMemberId, Integer taskId, String state, String workerId, String trace) {
this.kafkaClusterPhyId = kafkaClusterPhyId;
this.connectClusterId = connectClusterId;
this.connectorName = connectorName;
this.workerMemberId = workerMemberId;
this.taskId = taskId;
this.state = state;
this.workerId = workerId;
this.trace = trace;
}
}

19
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/config/ConnectConfigInfo.java Normal file
View File

@@ -0,0 +1,19 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo;
/**
* @see ConfigInfo
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigInfo {
private ConnectConfigKeyInfo definition;
private ConnectConfigValueInfo value;
}

62
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/config/ConnectConfigInfos.java Normal file
View File

@@ -0,0 +1,62 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo;
import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos;
import java.util.ArrayList;
import java.util.List;
/**
* @see ConfigInfos
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigInfos {
private String name;
private int errorCount;
private List<String> groups;
private List<ConnectConfigInfo> configs;
public ConnectConfigInfos(ConfigInfos configInfos) {
this.name = configInfos.name();
this.errorCount = configInfos.errorCount();
this.groups = configInfos.groups();
this.configs = new ArrayList<>();
for (ConfigInfo configInfo: configInfos.values()) {
ConnectConfigKeyInfo definition = new ConnectConfigKeyInfo();
definition.setName(configInfo.configKey().name());
definition.setType(configInfo.configKey().type());
definition.setRequired(configInfo.configKey().required());
definition.setDefaultValue(configInfo.configKey().defaultValue());
definition.setImportance(configInfo.configKey().importance());
definition.setDocumentation(configInfo.configKey().documentation());
definition.setGroup(configInfo.configKey().group());
definition.setOrderInGroup(configInfo.configKey().orderInGroup());
definition.setWidth(configInfo.configKey().width());
definition.setDisplayName(configInfo.configKey().displayName());
definition.setDependents(configInfo.configKey().dependents());
ConnectConfigValueInfo value = new ConnectConfigValueInfo();
value.setName(configInfo.configValue().name());
value.setValue(configInfo.configValue().value());
value.setRecommendedValues(configInfo.configValue().recommendedValues());
value.setErrors(configInfo.configValue().errors());
value.setVisible(configInfo.configValue().visible());
ConnectConfigInfo connectConfigInfo = new ConnectConfigInfo();
connectConfigInfo.setDefinition(definition);
connectConfigInfo.setValue(value);
this.configs.add(connectConfigInfo);
}
}
}

38
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/config/ConnectConfigKeyInfo.java Normal file
View File

@@ -0,0 +1,38 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigKeyInfo;
import java.util.List;
/**
* @see ConfigKeyInfo
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigKeyInfo {
private String name;
private String type;
private boolean required;
private String defaultValue;
private String importance;
private String documentation;
private String group;
private int orderInGroup;
private String width;
private String displayName;
private List<String> dependents;
}

27
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/config/ConnectConfigValueInfo.java Normal file
View File

@@ -0,0 +1,27 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.config;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigValueInfo;
import java.util.List;
/**
* @see ConfigValueInfo
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigValueInfo {
private String name;
private String value;
private List<String> recommendedValues;
private List<String> errors;
private boolean visible;
}

20
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/connector/KSAbstractConnectState.java Normal file
View File

@@ -0,0 +1,20 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector;
import com.alibaba.fastjson.annotation.JSONField;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
/**
* @see ConnectorStateInfo.AbstractState
*/
@Data
public abstract class KSAbstractConnectState {
private String state;
private String trace;
@JSONField(name="worker_id")
@JsonProperty("worker_id")
private String workerId;
}

48
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/connector/KSConnector.java Normal file
View File

@@ -0,0 +1,48 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector;
import lombok.Data;
import java.io.Serializable;
@Data
public class KSConnector implements Serializable {
/**
* Kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* connect集群ID
*/
private Long connectClusterId;
/**
* connector名称
*/
private String connectorName;
/**
* connector类名
*/
private String connectorClassName;
/**
* connector类型
*/
private String connectorType;
/**
* 访问过的Topic列表
*/
private String topics;
/**
* task数
*/
private Integer taskCount;
/**
* 状态
*/
private String state;
}

26
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/connector/KSConnectorInfo.java Normal file
View File

@@ -0,0 +1,26 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector;
import lombok.Data;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorType;
import org.apache.kafka.connect.util.ConnectorTaskId;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
/**
* copy from:
* @see org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo
*/
@Data
public class KSConnectorInfo implements Serializable {
private Long connectClusterId;
private String name;
private Map<String, String> config;
private List<ConnectorTaskId> tasks;
private ConnectorType type;
}

11
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/connector/KSConnectorState.java Normal file
View File

@@ -0,0 +1,11 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector;
import lombok.Data;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
/**
* @see ConnectorStateInfo.ConnectorState
*/
@Data
public class KSConnectorState extends KSAbstractConnectState {
}

21
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/connector/KSConnectorStateInfo.java Normal file
View File

@@ -0,0 +1,21 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector;
import lombok.Data;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorType;
import java.util.List;
/**
* @see ConnectorStateInfo
*/
@Data
public class KSConnectorStateInfo {
private String name;
private KSConnectorState connector;
private List<KSTaskState> tasks;
private ConnectorType type;
}

12
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/connector/KSTaskState.java Normal file
View File

@@ -0,0 +1,12 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector;
import lombok.Data;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo;
/**
* @see ConnectorStateInfo.TaskState
*/
@Data
public class KSTaskState extends KSAbstractConnectState {
private int id;
}

38
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/connect/plugin/ConnectPluginBasic.java Normal file
View File

@@ -0,0 +1,38 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.connect.plugin;
import com.alibaba.fastjson.annotation.JSONField;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.ApiModel;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
/**
* @author zengqiao
* @date 22/10/17
*/
@Data
@ApiModel(description = "Connect插件信息")
@NoArgsConstructor
public class ConnectPluginBasic implements Serializable {
/**
* Json序列化时对应的字段
*/
@JSONField(name="class")
@JsonProperty("class")
private String className;
private String type;
private String version;
private String helpDocLink;
public ConnectPluginBasic(String className, String type, String version, String helpDocLink) {
this.className = className;
this.type = type;
this.version = version;
this.helpDocLink = helpDocLink;
}
}

6
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/group/Group.java
View File

@@ -1,12 +1,12 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.group;
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafka.KSGroupDescription;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupTypeEnum;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.clients.admin.ConsumerGroupDescription;
import java.util.ArrayList;
import java.util.List;
@@ -61,9 +61,9 @@ public class Group {
*/
private int coordinatorId;
public Group(Long clusterPhyId, String groupName, ConsumerGroupDescription groupDescription) {
public Group(Long clusterPhyId, String groupName, KSGroupDescription groupDescription) {
this.clusterPhyId = clusterPhyId;
this.type = groupDescription.isSimpleConsumerGroup()? GroupTypeEnum.CONSUMER: GroupTypeEnum.CONNECTOR;
this.type = GroupTypeEnum.getTypeByProtocolType(groupDescription.protocolType());
this.name = groupName;
this.state = GroupStateEnum.getByRawState(groupDescription.state());
this.memberCount = groupDescription.members() == null ? 0 : groupDescription.members().size();

16
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/cluster/ConnectClusterParam.java Normal file
View File

@@ -0,0 +1,16 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author wyb
* @date 2022/11/9
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectClusterParam extends ClusterParam{
protected Long connectClusterId;
}

26
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/connect/ConnectorParam.java Normal file
View File

@@ -0,0 +1,26 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterParam;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ConnectClusterParam;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author wyb
* @date 2022/11/8
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectorParam extends ConnectClusterParam {
private String connectorName;
public ConnectorParam(Long connectClusterId, String connectorName) {
super(connectClusterId);
this.connectorName = connectorName;
}
}

21
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/metric/connect/ConnectClusterMetricParam.java Normal file
View File

@@ -0,0 +1,21 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.MetricParam;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author wyb
* @date 2022/11/1
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectClusterMetricParam extends MetricParam {
private Long connectClusterId;
private String metric;
}

29
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/param/metric/connect/ConnectorMetricParam.java Normal file
View File

@@ -0,0 +1,29 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.MetricParam;
import com.xiaojukeji.know.streaming.km.common.enums.connect.ConnectorTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author wyb
* @date 2022/11/2
*/
@Data
@NoArgsConstructor
public class ConnectorMetricParam extends MetricParam {
private Long connectClusterId;
private String connectorName;
private String metricName;
private ConnectorTypeEnum connectorType;
public ConnectorMetricParam(Long connectClusterId, String connectorName, String metricName, ConnectorTypeEnum connectorType) {
this.connectClusterId = connectClusterId;
this.connectorName = connectorName;
this.metricName = metricName;
this.connectorType = connectorType;
}
}

7
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/result/Result.java
View File

@@ -100,6 +100,13 @@ public class Result<T> extends BaseResult {
return result;
}
public static <T> Result<T> buildFrom(Result ret) {
Result<T> result = new Result<>();
result.setCode(ret.getCode());
result.setMessage(ret.getMessage());
return result;
}
public static <T> Result<T> buildFrom(ValidateKafkaAddressErrorEnum errorEnum, String msg) {
Result<T> result = new Result<>();
result.setCode(errorEnum.getCode());

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/result/ResultStatus.java
View File

@@ -54,6 +54,8 @@ public enum ResultStatus {
* 调用错误, [8000, 9000)
*/
KAFKA_OPERATE_FAILED(8010, "Kafka操作失败"),
KAFKA_CONNECTOR_OPERATE_FAILED(8011, "KafkaConnect操作失败"),
KAFKA_CONNECTOR_READ_FAILED(8012, "KafkaConnect读失败"),
MYSQL_OPERATE_FAILED(8020, "MySQL操作失败"),
ZK_OPERATE_FAILED(8030, "ZK操作失败"),
ZK_FOUR_LETTER_CMD_FORBIDDEN(8031, "ZK四字命令被禁止"),

13
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/version/VersionConnectJmxInfo.java Normal file
View File

@@ -0,0 +1,13 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.version;
import com.xiaojukeji.know.streaming.km.common.enums.connect.ConnectorTypeEnum;
import lombok.Data;
/**
* @author wyb
* @date 2022/11/24
*/
@Data
public class VersionConnectJmxInfo extends VersionJmxInfo{
private ConnectorTypeEnum type;
}

1
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/version/VersionControlItem.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.common.bean.entity.version;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/zookeeper/Znode.java
View File

@@ -1,7 +1,5 @@
package com.xiaojukeji.know.streaming.km.common.bean.entity.zookeeper;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.zookeeper.data.Stat;

27
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/event/cluster/connect/ConnectClusterLoadChangedEvent.java Normal file
View File

@@ -0,0 +1,27 @@
package com.xiaojukeji.know.streaming.km.common.bean.event.cluster.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
/**
* @author wyb
* @date 2022/11/7
*/
@Getter
public class ConnectClusterLoadChangedEvent extends ApplicationEvent {
private ConnectCluster inDBConnectCluster;
private ConnectCluster inCacheConnectCluster;
private final OperationEnum operationEnum;
public ConnectClusterLoadChangedEvent(Object source, ConnectCluster inDBConnectCluster, ConnectCluster inCacheConnectCluster, OperationEnum operationEnum) {
super(source);
this.inDBConnectCluster = inDBConnectCluster;
this.inCacheConnectCluster = inCacheConnectCluster;
this.operationEnum = operationEnum;
}
}

21
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/event/metric/connect/ConnectClusterMetricEvent.java Normal file
View File

@@ -0,0 +1,21 @@
package com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectClusterMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BaseMetricEvent;
import lombok.Getter;
import java.util.List;
/**
* @author wyb
* @date 2022/11/7
*/
@Getter
public class ConnectClusterMetricEvent extends BaseMetricEvent {
private List<ConnectClusterMetrics> connectClusterMetrics;
public ConnectClusterMetricEvent(Object source, List<ConnectClusterMetrics> connectClusterMetrics) {
super(source);
this.connectClusterMetrics = connectClusterMetrics;
}
}

21
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/event/metric/connect/ConnectorMetricEvent.java Normal file
View File

@@ -0,0 +1,21 @@
package com.xiaojukeji.know.streaming.km.common.bean.event.metric.connect;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectorMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.BaseMetricEvent;
import lombok.Getter;
import java.util.List;
/**
* @author wyb
* @date 2022/11/7
*/
@Getter
public class ConnectorMetricEvent extends BaseMetricEvent {
private List<ConnectorMetrics> connectorMetricsList;
public ConnectorMetricEvent(Object source, List<ConnectorMetrics> connectorMetricsList) {
super(source);
this.connectorMetricsList = connectorMetricsList;
}
}

51
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/connect/ConnectClusterPO.java Normal file
View File

@@ -0,0 +1,51 @@
package com.xiaojukeji.know.streaming.km.common.bean.po.connect;
import com.baomidou.mybatisplus.annotation.TableName;
import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import lombok.Data;
@Data
@TableName(Constant.MYSQL_KC_TABLE_NAME_PREFIX + "connect_cluster")
public class ConnectClusterPO extends BasePO {
/**
* Kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* 集群名字
*/
private String name;
/**
* 集群使用的消费组
*/
private String groupName;
/**
* 集群使用的消费组状态，也表示集群状态
*/
private Integer state;
/**
* 集群地址
*/
private String clusterUrl;
/**
* worker中显示的leader url信息
*/
private String memberLeaderUrl;
/**
* 版本信息
*/
private String version;
/**
* jmx配置
* @see com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig
*/
private String jmxProperties;
}

55
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/connect/ConnectWorkerPO.java Normal file
View File

@@ -0,0 +1,55 @@
package com.xiaojukeji.know.streaming.km.common.bean.po.connect;
import com.baomidou.mybatisplus.annotation.TableName;
import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import lombok.Data;
@Data
@TableName(Constant.MYSQL_KC_TABLE_NAME_PREFIX + "worker")
public class ConnectWorkerPO extends BasePO {
/**
* Kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* 集群ID
*/
private Long connectClusterId;
/**
* 成员ID
*/
private String memberId;
/**
* 主机
*/
private String host;
/**
* Jmx端口
*/
private Integer jmxPort;
/**
* URL
*/
private String url;
/**
* leader的URL
*/
private String leaderUrl;
/**
* 1：是leader，0：不是leader
*/
private Integer leader;
/**
* worker地址
*/
private String workerId;
}

50
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/connect/ConnectorPO.java Normal file
View File

@@ -0,0 +1,50 @@
package com.xiaojukeji.know.streaming.km.common.bean.po.connect;
import com.baomidou.mybatisplus.annotation.TableName;
import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import lombok.Data;
@Data
@TableName(Constant.MYSQL_KC_TABLE_NAME_PREFIX + "connector")
public class ConnectorPO extends BasePO {
/**
* Kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* connect集群ID
*/
private Long connectClusterId;
/**
* connector名称
*/
private String connectorName;
/**
* connector类名
*/
private String connectorClassName;
/**
* connector类型
*/
private String connectorType;
/**
* 访问过的Topic列表
*/
private String topics;
/**
* task数
*/
private Integer taskCount;
/**
* 状态
*/
private String state;
}

45
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/connect/WorkerConnectorPO.java Normal file
View File

@@ -0,0 +1,45 @@
package com.xiaojukeji.know.streaming.km.common.bean.po.connect;
import com.baomidou.mybatisplus.annotation.TableName;
import com.xiaojukeji.know.streaming.km.common.bean.po.BasePO;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import lombok.Data;
@Data
@TableName(Constant.MYSQL_KC_TABLE_NAME_PREFIX + "worker_connector")
public class WorkerConnectorPO extends BasePO {
/**
* connect集群ID
*/
private Long connectClusterId;
/**
* kafka集群ID
*/
private Long kafkaClusterPhyId;
/**
* connector名称
*/
private String connectorName;
/**
* worker成员ID
*/
private String workerMemberId;
/**
* 任务ID
*/
private Integer taskId;
/**
* task状态
*/
private String state;
/**
* worker信息
*/
private String workerId;
}

18
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connect/ConnectClusterBasicCombineExistVO.java Normal file
View File

@@ -0,0 +1,18 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 集群的Connect集群信息
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Connect集群基本信息")
public class ConnectClusterBasicCombineExistVO extends ConnectClusterBasicVO {
@ApiModelProperty(value="是否存在", example = "true")
protected Boolean exist;
}

44
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connect/ConnectClusterBasicVO.java Normal file
View File

@@ -0,0 +1,44 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connect;
import com.xiaojukeji.know.streaming.km.common.bean.vo.BaseVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 集群的Connect集群信息
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Connect集群基本信息")
public class ConnectClusterBasicVO extends BaseVO {
@ApiModelProperty(value = "Connect集群ID", example = "1")
private Long id;
@ApiModelProperty(value = "Connect集群名称", example = "know-streaming")
private String name;
@ApiModelProperty(value = "Connect集群使用的Group", example = "know-streaming")
private String groupName;
@ApiModelProperty(value = "Connect集群URL", example = "http://127.0.0.1:8080")
private String clusterUrl;
@ApiModelProperty(value = "Connect集群获取到的URL", example = "http://127.0.0.1:8080")
private String memberLeaderUrl;
@ApiModelProperty(value = "Connect集群版本", example = "2.5.1")
private String version;
@ApiModelProperty(value = "JMX配置", example = "")
private String jmxProperties;
/**
* 集群使用的消费组状态，也表示集群状态
* @see com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum
*/
@ApiModelProperty(value = "状态，2表示Dead，只有Dead才可以删除", example = "")
private Integer state;
}

42
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connect/ConnectStateVO.java Normal file
View File

@@ -0,0 +1,42 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connect;
import com.xiaojukeji.know.streaming.km.common.bean.vo.BaseVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 集群Connectors状态信息
* @author zengqiao
* @date 22/10/17
*/
@Data
@ApiModel(description = "集群Connects状态信息")
public class ConnectStateVO extends BaseVO {
@ApiModelProperty(value = "健康检查状态", example = "1")
private Integer healthState;
@ApiModelProperty(value = "健康检查通过数", example = "1")
private Integer healthCheckPassed;
@ApiModelProperty(value = "健康检查总数", example = "1")
private Integer healthCheckTotal;
@ApiModelProperty(value = "connect集群数", example = "1")
private Integer connectClusterCount;
@ApiModelProperty(value = "worker数", example = "1")
private Integer workerCount;
@ApiModelProperty(value = "总Connector数", example = "1")
private Integer totalConnectorCount;
@ApiModelProperty(value = "存活Connector数", example = "1")
private Integer aliveConnectorCount;
@ApiModelProperty(value = "总Task数", example = "1")
private Integer totalTaskCount;
@ApiModelProperty(value = "存活Task数", example = "1")
private Integer aliveTaskCount;
}

42
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connector/ClusterConnectorOverviewVO.java Normal file
View File

@@ -0,0 +1,42 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricLineVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
/**
* 集群Connector信息
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Connector概览信息")
public class ClusterConnectorOverviewVO extends ConnectorBasicVO {
@ApiModelProperty(value = "Connector插件名称", example = "know-streaming")
private String connectorClassName;
@ApiModelProperty(value = "Connector类型", example = "source")
private String connectorType;
/**
* @see org.apache.kafka.connect.runtime.AbstractStatus.State
*/
@ApiModelProperty(value = "状态", example = "RUNNING")
private String state;
@ApiModelProperty(value = "Task数", example = "100")
private Integer taskCount;
@ApiModelProperty(value = "访问的Topic列表", example = "")
private List<String> topicNameList;
@ApiModelProperty(value = "多个指标的当前值, 包括健康分/LogSize等")
private BaseMetrics latestMetrics;
@ApiModelProperty(value = "多个指标的历史曲线值，包括LogSize/BytesIn等")
private List<MetricLineVO> metricLines;
}

31
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connector/ClusterWorkerOverviewVO.java Normal file
View File

@@ -0,0 +1,31 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector;
import com.xiaojukeji.know.streaming.km.common.bean.vo.BaseVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 集群Worker信息
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Worker概览信息")
public class ClusterWorkerOverviewVO extends BaseVO {
@ApiModelProperty(value = "Connect集群ID", example = "1")
private Long connectClusterId;
@ApiModelProperty(value = "Connect集群名称", example = "know-streaming")
private String connectClusterName;
@ApiModelProperty(value = "worker主机", example = "know-streaming")
private String workerHost;
@ApiModelProperty(value = "Connector数", example = "10")
private Integer connectorCount;
@ApiModelProperty(value = "Task数", example = "10")
private Integer taskCount;
}

18
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connector/ConnectorBasicCombineExistVO.java Normal file
View File

@@ -0,0 +1,18 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 集群Connector信息
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Connector基本信息")
public class ConnectorBasicCombineExistVO extends ConnectorBasicVO {
@ApiModelProperty(value="是否存在", example = "true")
protected Boolean exist;
}

25
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/cluster/connector/ConnectorBasicVO.java Normal file
View File

@@ -0,0 +1,25 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector;
import com.xiaojukeji.know.streaming.km.common.bean.vo.BaseVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* 集群Connector信息
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Connector基本信息")
public class ConnectorBasicVO extends BaseVO {
@ApiModelProperty(value = "Connect集群ID", example = "1")
private Long connectClusterId;
@ApiModelProperty(value = "Connect集群名称", example = "know-streaming")
private String connectClusterName;
@ApiModelProperty(value = "Connector名称", example = "know-streaming")
private String connectorName;
}

32
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/connector/ConnectorStateVO.java Normal file
View File

@@ -0,0 +1,32 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.connector;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author wyb
* @date 2022/11/15
*/
@Data
public class ConnectorStateVO {
@ApiModelProperty(value = "connect集群ID", example = "1")
private Long connectClusterId;
@ApiModelProperty(value = "connector名称", example = "input1")
private String name;
@ApiModelProperty(value = "connector类型", example = "source")
private String type;
@ApiModelProperty(value = "connector状态", example = "running")
private String state;
@ApiModelProperty(value = "总Task数", example = "1")
private Integer totalTaskCount;
@ApiModelProperty(value = "存活Task数", example = "1")
private Integer aliveTaskCount;
@ApiModelProperty(value = "总Worker数", example = "1")
private Integer totalWorkerCount;
}

19
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/plugin/ConnectConfigInfoVO.java Normal file
View File

@@ -0,0 +1,19 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo;
/**
* @see ConfigInfo
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigInfoVO {
private ConnectConfigKeyInfoVO definition;
private ConnectConfigValueInfoVO value;
}

25
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/plugin/ConnectConfigInfosVO.java Normal file
View File

@@ -0,0 +1,25 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos;
import java.util.List;
/**
* @see ConfigInfos
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigInfosVO {
private String name;
private int errorCount;
private List<String> groups;
private List<ConnectConfigInfoVO> configs;
}

38
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/plugin/ConnectConfigKeyInfoVO.java Normal file
View File

@@ -0,0 +1,38 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigKeyInfo;
import java.util.List;
/**
* @see ConfigKeyInfo
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigKeyInfoVO {
private String name;
private String type;
private boolean required;
private String defaultValue;
private String importance;
private String documentation;
private String group;
private int orderInGroup;
private String width;
private String displayName;
private List<String> dependents;
}

27
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/plugin/ConnectConfigValueInfoVO.java Normal file
View File

@@ -0,0 +1,27 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.kafka.connect.runtime.rest.entities.ConfigValueInfo;
import java.util.List;
/**
* @see ConfigValueInfo
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ConnectConfigValueInfoVO {
private String name;
private String value;
private List<String> recommendedValues;
private List<String> errors;
private boolean visible;
}

26
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/plugin/ConnectPluginBasicVO.java Normal file
View File

@@ -0,0 +1,26 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.plugin;
import com.xiaojukeji.know.streaming.km.common.bean.vo.BaseVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author zengqiao
* @date 22/10/17
*/
@Data
@ApiModel(description = "Connect插件信息")
public class ConnectPluginBasicVO extends BaseVO {
@ApiModelProperty(value = "指标或操作项名称", example = "org.apache.kafka.connect.file.FileStreamSinkConnector")
private String className;
@ApiModelProperty(value = "类型", example = "source|sink")
private String type;
@ApiModelProperty(value = "版本", example = "2.5.1")
private String version;
@ApiModelProperty(value = "帮助文档地址", example = "")
private String helpDocLink;
}

32
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/connect/task/KCTaskOverviewVO.java Normal file
View File

@@ -0,0 +1,32 @@
package com.xiaojukeji.know.streaming.km.common.bean.vo.connect.task;
import com.xiaojukeji.know.streaming.km.common.bean.vo.BaseVO;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* Task信息概览
* @author zengqiao
* @date 22/02/23
*/
@Data
@ApiModel(description = "Task信息概览")
public class KCTaskOverviewVO extends BaseVO {
@ApiModelProperty(value = "connect集群ID", example = "1")
private Long connectClusterId;
@ApiModelProperty(value = "taskId", example = "1")
private Integer taskId;
@ApiModelProperty(value = "worker地址", example = "127.0.0.1:8080")
private String workerId;
@ApiModelProperty(value = "task状态", example = "RUNNING")
private String state;
@ApiModelProperty(value = "错误原因", example = "asx")
private String trace;
}

3
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/health/HealthCheckConfigVO.java
View File

@@ -20,6 +20,9 @@ public class HealthCheckConfigVO {
@ApiModelProperty(value="检查维度名称", example = "Broker")
private String dimensionName;
@ApiModelProperty(value="检查维度前端展示名称", example = "Connector")
private String dimensionDisplayName;
@ApiModelProperty(value="配置组", example = "HEALTH")
private String configGroup;

3
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/health/HealthScoreBaseResultVO.java
View File

@@ -21,6 +21,9 @@ public class HealthScoreBaseResultVO extends BaseTimeVO {
@ApiModelProperty(value="检查维度名称", example = "cluster")
private String dimensionName;
@ApiModelProperty(value="检查维度前端显示名称", example = "cluster")
private String dimensionDisplayName;
@ApiModelProperty(value="检查名称", example = "Group延迟")
private String configName;

30
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/component/RestTemplateConfig.java
View File

@@ -131,13 +131,22 @@ public class RestTemplateConfig {
}
} catch (Exception e) {
RESP_LOGGER.warn(
"method=traceResponse||code={}||url={}||text={}||headers={}||body={}||error={}||timeCost={}||subFlag={}",
response.getStatusCode(), url, response.getStatusText(), response.getHeaders(),
inputStringBuilder.toString(), e, (System.nanoTime() - nanoTime) / 1000 / 1000, subFlag);
"method=traceResponse||remoteResponse||code={}||url={}||text={}||headers={}||body={}||error={}||timeCost={}||subFlag={}",
response.getStatusCode(),
url,
response.getStatusText(),
response.getHeaders(),
inputStringBuilder.toString(),
e,
(System.nanoTime() - nanoTime) / 1000 / 1000,
subFlag
);
if (!response.getStatusCode().is2xxSuccessful()) {
throw new ThirdPartRemoteException(e.getMessage(), e, ResultStatus.HTTP_REQ_ERROR);
throw new ThirdPartRemoteException(getResponseBodyAndIgnoreException(response), e, ResultStatus.HTTP_REQ_ERROR);
}
}
String responseString = inputStringBuilder.toString().replace("\n", "");
responseString = responseString.substring(0, Math.min(responseString.length(), 5000));
@@ -172,6 +181,19 @@ public class RestTemplateConfig {
}
private String getResponseBodyAndIgnoreException(ClientHttpResponse response) {
try {
byte[] bytes = new byte[response.getBody().available()];
response.getBody().read(bytes);
return new String(bytes);
} catch (Exception e) {
// ignore
}
return "";
}
private static String simpleUrl(HttpRequest request) {
String url = request.getURI().toString();
int index = url.indexOf("?");

74
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/component/RestTool.java
View File

@@ -13,6 +13,7 @@ import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
/**
@@ -22,7 +23,6 @@ import java.util.Map;
*/
@Component
public class RestTool {
private static final ILog LOGGER = LogFactory.getLog(RestTool.class);
@Autowired
@@ -38,39 +38,38 @@ public class RestTool {
* @return
*/
public <T> T postObjectWithRawContent(String url, Object postBody, HttpHeaders headers, Class<T> resultType) {
ResponseEntity<String> result = restTemplate.exchange(url, HttpMethod.POST, new HttpEntity<>(postBody, headers),
String.class);
ResponseEntity<String> result = restTemplate.exchange(
url,
HttpMethod.POST,
new HttpEntity<>(postBody, headers),
String.class
);
return ConvertUtil.toObj(result.getBody(), resultType);
}
/**
* POST请求
* @param url 请求地址
* @param request 请求内容
* @param responseType 期望返回的类型
* @param postBody 请求内容
* @param resultType 期望返回的类型
* @param <T> 泛型T
* @return T
*/
public <T> T postObjectWithJsonContent(String url, Object request, Type responseType) {
HttpHeaders jsonHead = getJsonContentHeaders();
ResponseEntity<String> result = restTemplate.exchange(url, HttpMethod.POST,
new HttpEntity<Object>( ConvertUtil.obj2Json(request), jsonHead), String.class);
return ConvertUtil.toObj(result.getBody(), responseType);
public <T> T postObjectWithJsonContent(String url, Object postBody, Class<T> resultType) {
return this.postObjectWithRawContent(url, postBody, this.getJsonContentHeaders(), resultType);
}
/**
* POST请求
* @param url 请求地址
* @param request 请求内容
* @param responseType 期望返回的类型
* @param postBody 请求内容
* @param resultType 期望返回的类型
* @param <T> 泛型T
* @return T
*/
public <T> T postObjectWithJsonContentAndHeader(String url, Map<String, String> headers, Object request,
Type responseType) {
ResponseEntity<String> result = restTemplate.exchange(url, HttpMethod.POST,
new HttpEntity<Object>(ConvertUtil.obj2Json(request), getJsonContentHeaders(headers)), String.class);
return ConvertUtil.toObj(result.getBody(), responseType);
public <T> T postObjectWithJsonContentAndHeader(String url, Map<String, String> headers, Object postBody, Class<T> resultType) {
return this.postObjectWithRawContent(url, postBody, this.getJsonContentHeaders(headers), resultType);
}
/**
@@ -81,8 +80,15 @@ public class RestTool {
* @param <T> 泛型T
* @return T
*/
public <T> T getObjectWithJsonContent(String url, Map<String, ?> params, Type resultType) {
ResponseEntity<String> result = restTemplate.exchange(url, HttpMethod.GET, null, String.class, params);
public <T> T getObjectWithJsonContent(String url, Map<String, ?> params, Class<T> resultType) {
ResponseEntity<String> result = restTemplate.exchange(
url,
HttpMethod.GET,
null,
String.class,
params
);
return ConvertUtil.toObj(result.getBody(), resultType);
}
@@ -95,8 +101,13 @@ public class RestTool {
* @return T
*/
public <T> T getForObject(String url, HttpHeaders headers, Type resultType) {
ResponseEntity<String> result = restTemplate.exchange(url, HttpMethod.GET, new HttpEntity<>(null, headers),
String.class);
ResponseEntity<String> result = restTemplate.exchange(
url,
HttpMethod.GET,
new HttpEntity<>(null, headers),
String.class
);
return ConvertUtil.toObj(result.getBody(), resultType);
}
@@ -169,6 +180,26 @@ public class RestTool {
return result.getBody();
}
/**
* GET请求
* @param url 请求地址
* @param params 请求参数
* @param resultType 返回类型
* @param <T> 泛型T
* @return T
*/
public <T> List<T> getArrayObjectWithJsonContent(String url, Map<String, ?> params, Class<T> resultType) {
ResponseEntity<String> result = restTemplate.exchange(
url,
HttpMethod.GET,
null,
String.class,
params
);
return ConvertUtil.str2ObjArrayByJson(result.getBody(), resultType);
}
/**
* 根据map中的参数构建url+queryString
* @param url 请求地址
@@ -181,7 +212,6 @@ public class RestTool {
}
UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(url);
return builder.toUriString();
}

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/ApiPrefix.java
View File

@@ -12,6 +12,8 @@ public class ApiPrefix {
public static final String API_V3_PREFIX = API_PREFIX + "v3/";
public static final String API_V3_CONNECT_PREFIX = API_V3_PREFIX + "kafka-connect/";
public static final String API_V3_OPEN_PREFIX = API_V3_PREFIX + "open/";
private ApiPrefix() {

1
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/Constant.java
View File

@@ -45,6 +45,7 @@ public class Constant {
public static final int INVALID_CODE = -1;
public static final String MYSQL_TABLE_NAME_PREFIX = "ks_km_";
public static final String MYSQL_KC_TABLE_NAME_PREFIX = "ks_kc_";
public static final String SWAGGER_API_TAG_PREFIX = "KS-KM-";

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/KafkaConstant.java
View File

@@ -43,6 +43,8 @@ public class KafkaConstant {
public static final String CONTROLLER_ROLE = "controller";
public static final String DEFAULT_CONNECT_VERSION = "2.5.0";
public static final Map<String, ConfigDef.ConfigKey> KAFKA_ALL_CONFIG_DEF_MAP = new ConcurrentHashMap<>();
static {

15
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/MsgConstant.java
View File

@@ -95,4 +95,19 @@ public class MsgConstant {
public static String getJobNotExist(Long jobId) {
return String.format("jobId:[%d] 不存在", jobId);
}
/**************************************************** Connect-Cluster ****************************************************/
public static String getConnectClusterBizStr(Long clusterId, String clusterName){
return String.format("Connect集群ID:[%d] 集群名称:[%s]", clusterId, clusterName);
}
public static String getConnectClusterNotExist(Long clusterId) {
return String.format("Connect集群ID:[%d] 不存在或者未加载", clusterId);
}
public static String getConnectorBizStr(Long clusterPhyId, String topicName) {
return String.format("Connect集群ID:[%d] Connector名称:[%s]", clusterPhyId, topicName);
}
}

15
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/connect/KafkaConnectConstant.java Normal file
View File

@@ -0,0 +1,15 @@
package com.xiaojukeji.know.streaming.km.common.constant.connect;
/**
* @author zengqiao
* @date 20/5/20
*/
public class KafkaConnectConstant {
public static final String CONNECTOR_CLASS_FILED_NAME = "connector.class";
public static final String CONNECTOR_TOPICS_FILED_NAME = "topics";
public static final String CONNECTOR_TOPICS_FILED_ERROR_VALUE = "know-streaming-connect-illegal-value";
private KafkaConnectConstant() {
}
}

142
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/converter/ConnectConverter.java Normal file
View File

@@ -0,0 +1,142 @@
package com.xiaojukeji.know.streaming.km.common.converter;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.ConnectCluster;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector.KSConnector;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector.KSConnectorInfo;
import com.xiaojukeji.know.streaming.km.common.bean.entity.connect.connector.KSConnectorStateInfo;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect.ConnectorMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.po.connect.ConnectorPO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connect.ConnectClusterBasicCombineExistVO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector.ClusterConnectorOverviewVO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector.ConnectorBasicCombineExistVO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.cluster.connector.ConnectorBasicVO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricLineVO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
import com.xiaojukeji.know.streaming.km.common.constant.connect.KafkaConnectConstant;
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
public class ConnectConverter {
public static ConnectorBasicCombineExistVO convert2BasicVO(ConnectCluster connectCluster, ConnectorPO connectorPO) {
ConnectorBasicCombineExistVO vo = new ConnectorBasicCombineExistVO();
if (connectCluster == null || connectorPO == null) {
vo.setExist(false);
return vo;
}
vo.setExist(true);
vo.setConnectClusterId(connectorPO.getConnectClusterId());
vo.setConnectClusterName(connectCluster.getName());
vo.setConnectorName(connectorPO.getConnectorName());
return vo;
}
public static List<ConnectorBasicVO> convert2BasicVOList(
List<ConnectCluster> clusterList,
List<ConnectorPO> poList) {
Map<Long, ConnectCluster> clusterMap = new HashMap<>();
clusterList.stream().forEach(elem -> clusterMap.put(elem.getId(), elem));
List<ConnectorBasicVO> voList = new ArrayList<>();
poList.stream().filter(item -> clusterMap.containsKey(item.getConnectClusterId())).forEach(elem -> {
ConnectorBasicVO vo = new ConnectorBasicVO();
vo.setConnectClusterId(elem.getConnectClusterId());
vo.setConnectClusterName(clusterMap.get(elem.getConnectClusterId()).getName());
vo.setConnectorName(elem.getConnectorName());
voList.add(vo);
});
return voList;
}
public static ConnectClusterBasicCombineExistVO convert2ConnectClusterBasicCombineExistVO(ConnectCluster connectCluster) {
if (connectCluster == null) {
ConnectClusterBasicCombineExistVO combineExistVO = new ConnectClusterBasicCombineExistVO();
combineExistVO.setExist(false);
return combineExistVO;
}
ConnectClusterBasicCombineExistVO combineExistVO = ConvertUtil.obj2Obj(connectCluster, ConnectClusterBasicCombineExistVO.class);
combineExistVO.setExist(true);
return combineExistVO;
}
public static List<ClusterConnectorOverviewVO> convert2ClusterConnectorOverviewVOList(List<ConnectCluster> clusterList,
List<ConnectorPO> poList,
List<ConnectorMetrics> metricsList) {
Map<Long, ConnectCluster> clusterMap = new HashMap<>();
clusterList.stream().forEach(elem -> clusterMap.put(elem.getId(), elem));
Map<String, ConnectorMetrics> metricMap = metricsList.stream().collect(Collectors.toMap(elem -> elem.getConnectClusterId() + "@" + elem.getConnectorName(), Function.identity()));
List<ClusterConnectorOverviewVO> voList = new ArrayList<>();
poList.stream().filter(item -> clusterMap.containsKey(item.getConnectClusterId())).forEach(elem -> {
ClusterConnectorOverviewVO vo = new ClusterConnectorOverviewVO();
vo.setConnectClusterId(elem.getConnectClusterId());
vo.setConnectClusterName(clusterMap.get(elem.getConnectClusterId()).getName());
vo.setConnectorName(elem.getConnectorName());
vo.setConnectorClassName(elem.getConnectorClassName());
vo.setConnectorType(elem.getConnectorType());
vo.setState(elem.getState());
vo.setTaskCount(elem.getTaskCount());
vo.setTopicNameList(CommonUtils.string2StrList(elem.getTopics()));
vo.setLatestMetrics(metricMap.getOrDefault(elem.getConnectClusterId() + "@" + elem.getConnectorName(), new ConnectorMetrics(elem.getConnectClusterId(), elem.getConnectorName())));
voList.add(vo);
});
return voList;
}
public static List<ClusterConnectorOverviewVO> supplyData2ClusterConnectorOverviewVOList(List<ClusterConnectorOverviewVO> voList,
List<MetricMultiLinesVO> metricLineVOList) {
Map<String, List<MetricLineVO>> metricLineMap = new HashMap<>();
if (metricLineVOList != null) {
for (MetricMultiLinesVO metricMultiLinesVO : metricLineVOList) {
metricMultiLinesVO.getMetricLines()
.forEach(metricLineVO -> {
String key = metricLineVO.getName();
List<MetricLineVO> metricLineVOS = metricLineMap.getOrDefault(key, new ArrayList<>());
metricLineVOS.add(metricLineVO);
metricLineMap.put(key, metricLineVOS);
});
}
}
voList.forEach(elem -> {
elem.setMetricLines(metricLineMap.get(genConnectorKey(elem.getConnectClusterId(), elem.getConnectorName())));
});
return voList;
}
public static KSConnector convert2KSConnector(Long kafkaClusterPhyId, Long connectClusterId, KSConnectorInfo connectorInfo, KSConnectorStateInfo stateInfo, List<String> topicNameList) {
KSConnector ksConnector = new KSConnector();
ksConnector.setKafkaClusterPhyId(kafkaClusterPhyId);
ksConnector.setConnectClusterId(connectClusterId);
ksConnector.setConnectorName(connectorInfo.getName());
ksConnector.setConnectorClassName(connectorInfo.getConfig().getOrDefault(KafkaConnectConstant.CONNECTOR_CLASS_FILED_NAME, ""));
ksConnector.setConnectorType(connectorInfo.getType().name());
ksConnector.setTopics(topicNameList != null? CommonUtils.strList2String(topicNameList): "");
ksConnector.setTaskCount(connectorInfo.getTasks() != null? connectorInfo.getTasks().size(): 0);
ksConnector.setState(stateInfo != null? stateInfo.getConnector().getState(): "");
return ksConnector;
}
private static String genConnectorKey(Long connectorId, String connectorName){
return connectorId + "#" + connectorName;
}
private ConnectConverter() {
}
}

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/converter/HealthScoreVOConverter.java
View File

@@ -21,6 +21,7 @@ public class HealthScoreVOConverter {
HealthScoreResultDetailVO vo = new HealthScoreResultDetailVO();
vo.setDimension(healthScoreResult.getCheckNameEnum().getDimensionEnum().getDimension());
vo.setDimensionName(healthScoreResult.getCheckNameEnum().getDimensionEnum().getMessage());
vo.setDimensionDisplayName(healthScoreResult.getCheckNameEnum().getDimensionEnum().getDimensionDisplayName());
vo.setConfigName(healthScoreResult.getCheckNameEnum().getConfigName());
vo.setConfigItem(healthScoreResult.getCheckNameEnum().getConfigItem());
vo.setConfigDesc(healthScoreResult.getCheckNameEnum().getConfigDesc());
@@ -63,6 +64,7 @@ public class HealthScoreVOConverter {
public static HealthCheckConfigVO convert2HealthCheckConfigVO(String groupName, BaseClusterHealthConfig config) {
HealthCheckConfigVO vo = new HealthCheckConfigVO();
vo.setDimensionCode(config.getCheckNameEnum().getDimensionEnum().getDimension());
vo.setDimensionDisplayName(config.getCheckNameEnum().getDimensionEnum().getDimensionDisplayName());
vo.setDimensionName(config.getCheckNameEnum().getDimensionEnum().name());
vo.setConfigGroup(groupName);
vo.setConfigName(config.getCheckNameEnum().getConfigName());

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/converter/TopicVOConverter.java
View File

@@ -1,6 +1,5 @@
package com.xiaojukeji.know.streaming.km.common.converter;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
@@ -14,7 +13,6 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiL
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.partition.TopicPartitionVO;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import io.swagger.annotations.ApiModelProperty;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.header.Header;

32
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/connect/ConnectActionEnum.java Normal file
View File

@@ -0,0 +1,32 @@
package com.xiaojukeji.know.streaming.km.common.enums.connect;
public enum ConnectActionEnum {
/**
*
*/
STOP(2, "stop"),
RESUME(3,"resume"),
RESTART(4,"restart"),
UNKNOWN(-1, "unknown");
ConnectActionEnum(int status, String value) {
this.status = status;
this.value = value;
}
private final int status;
private final String value;
public int getStatus() {
return status;
}
public String getValue() {
return value;
}
}

32
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/connect/ConnectorTypeEnum.java Normal file
View File

@@ -0,0 +1,32 @@
package com.xiaojukeji.know.streaming.km.common.enums.connect;
/**
* @author wyb
* @date 2022/11/25
*/
public enum ConnectorTypeEnum {
UNKNOWN(-1, "unknown"),
SOURCE(1, "source"),
SINK(2, "sink");
private final int code;
private final String value;
ConnectorTypeEnum(int code, String value) {
this.code = code;
this.value = value;
}
public static ConnectorTypeEnum getByName(String name) {
for (ConnectorTypeEnum typeEnum : ConnectorTypeEnum.values()) {
if (typeEnum.name().equals(name)) {
return typeEnum;
}
}
return UNKNOWN;
}
}

24
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/group/GroupTypeEnum.java
View File

@@ -2,6 +2,7 @@ package com.xiaojukeji.know.streaming.km.common.enums.group;
import lombok.Getter;
/**
* @author wyb
* @date 2022/10/11
@@ -13,12 +14,18 @@ public enum GroupTypeEnum {
CONSUMER(0, "Consumer客户端的消费组"),
CONNECTOR(1, "Connector的消费组");
CONNECTOR(1, "Connector的消费组"),
CONNECT_CLUSTER(2, "Connect集群");
private final Integer code;
private final String msg;
public static final String CONNECTOR_PROTOCOL_TYPE = "consumer";
public static final String CONNECT_CLUSTER_PROTOCOL_TYPE = "connect";
GroupTypeEnum(Integer code, String msg) {
this.code = code;
this.msg = msg;
@@ -33,4 +40,19 @@ public enum GroupTypeEnum {
}
return UNKNOWN;
}
public static GroupTypeEnum getTypeByProtocolType(String protocolType) {
if (protocolType == null) {
return UNKNOWN;
}
if (protocolType.isEmpty()) {
return CONSUMER;
} else if (CONNECTOR_PROTOCOL_TYPE.equals(protocolType)) {
return CONNECTOR;
} else if (CONNECT_CLUSTER_PROTOCOL_TYPE.equals(protocolType)) {
return CONNECT_CLUSTER;
} else {
return UNKNOWN;
}
}
}

23
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/health/HealthCheckDimensionEnum.java
View File

@@ -8,19 +8,23 @@ import lombok.Getter;
*/
@Getter
public enum HealthCheckDimensionEnum {
UNKNOWN(-1, "未知"),
UNKNOWN(-1, "未知", "未知"),
CLUSTER(0, "Cluster"),
CLUSTER(0, "Cluster", "Cluster"),
BROKER(1, "Broker"),
BROKER(1, "Broker", "Broker"),
TOPIC(2, "Topic"),
TOPIC(2, "Topic", "Topic"),
GROUP(3, "Group"),
GROUP(3, "Group", "Group"),
ZOOKEEPER(4, "Zookeeper"),
ZOOKEEPER(4, "Zookeeper", "Zookeeper"),
MAX_VAL(100, "所有的dimension的值需要小于MAX_VAL")
CONNECT_CLUSTER(5, "ConnectCluster", "Connect"),
CONNECTOR(6, "Connector", "Connect"),
MAX_VAL(100, "所有的dimension的值需要小于MAX_VAL", "Ignore")
;
@@ -28,9 +32,12 @@ public enum HealthCheckDimensionEnum {
private final String message;
HealthCheckDimensionEnum(int dimension, String message) {
private final String dimensionDisplayName;
HealthCheckDimensionEnum(int dimension, String message, String dimensionDisplayName) {
this.dimension = dimension;
this.message = message;
this.dimensionDisplayName=dimensionDisplayName;
}
public static HealthCheckDimensionEnum getByCode(Integer dimension) {

27
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/health/HealthCheckNameEnum.java
View File

@@ -132,6 +132,33 @@ public enum HealthCheckNameEnum {
false
),
CONNECT_CLUSTER_TASK_STARTUP_FAILURE_PERCENTAGE(
HealthCheckDimensionEnum.CONNECT_CLUSTER,
"TaskStartupFailurePercentage",
Constant.HC_CONFIG_NAME_PREFIX+"CONNECT_CLUSTER_TASK_STARTUP_FAILURE_PERCENTAGE",
"connect集群任务启动失败概率",
HealthCompareValueConfig.class,
false
),
CONNECTOR_FAILED_TASK_COUNT(
HealthCheckDimensionEnum.CONNECTOR,
"ConnectorFailedTaskCount",
Constant.HC_CONFIG_NAME_PREFIX+"CONNECTOR_FAILED_TASK_COUNT",
"connector失败状态的任务数量",
HealthCompareValueConfig.class,
false
),
CONNECTOR_UNASSIGNED_TASK_COUNT(
HealthCheckDimensionEnum.CONNECTOR,
"ConnectorUnassignedTaskCount",
Constant.HC_CONFIG_NAME_PREFIX+"CONNECTOR_UNASSIGNED_TASK_COUNT",
"connector未被分配的任务数量",
HealthCompareValueConfig.class,
false
)
;

3
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/ModuleEnum.java
View File

@@ -33,6 +33,9 @@ public enum ModuleEnum {
KAFKA_CONTROLLER(70, "KafkaController"),
KAFKA_CONNECT_CLUSTER(80, "KafkaConnectCluster"),
KAFKA_CONNECT_CONNECTOR(81, "KafkaConnectConnector"),
PLATFORM_CONFIG(100, "平台配置"),
JOB_KAFKA_REPLICA_REASSIGN(110, "Job-KafkaReplica迁移"),

2
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/operaterecord/OperationEnum.java
View File

@@ -30,6 +30,8 @@ public enum OperationEnum {
CANCEL(10, "取消"),
RESTART(11, "重启"),
;
OperationEnum(int code, String desc) {

7
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/enums/version/VersionItemTypeEnum.java
View File

@@ -13,6 +13,10 @@ public enum VersionItemTypeEnum {
METRIC_ZOOKEEPER(110, "ZookeeperMetric"),
METRIC_CONNECT_CLUSTER(120, "ConnectClusterMetric"),
METRIC_CONNECT_CONNECTOR(121, "ConnectConnectorMetric"),
METRIC_CONNECT_MIRROR_MAKER(122, "ConnectMirrorMakerMetric"),
/**
* 服务端查询
*/
@@ -37,6 +41,9 @@ public enum VersionItemTypeEnum {
SERVICE_OP_REASSIGNMENT(330, "service_reassign_operation"),
SERVICE_OP_CONNECT_CLUSTER(400, "service_connect_cluster_operation"),
SERVICE_OP_CONNECT_CONNECTOR(401, "service_connect_connector_operation"),
SERVICE_OP_CONNECT_PLUGIN(402, "service_connect_plugin_operation"),
/**
* 前端操作

110
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxAttribute.java
View File

@@ -34,6 +34,116 @@ public class JmxAttribute {
public static final String VERSION = "Version";
/*********************************************************** connect cluster***********************************************************/
public static final String TASK_COUNT = "task-count";
public static final String CONNECTOR_STARTUP_ATTEMPTS_TOTAL = "connector-startup-attempts-total";
public static final String CONNECTOR_STARTUP_FAILURE_PERCENTAGE = "connector-startup-failure-percentage";
public static final String CONNECTOR_STARTUP_FAILURE_TOTAL = "connector-startup-failure-total";
public static final String CONNECTOR_STARTUP_SUCCESS_PERCENTAGE = "connector-startup-success-percentage";
public static final String CONNECTOR_STARTUP_SUCCESS_TOTAL = "connector-startup-success-total";
public static final String TASK_STARTUP_ATTEMPTS_TOTAL = "task-startup-attempts-total";
public static final String TASK_STARTUP_FAILURE_PERCENTAGE = "task-startup-failure-percentage";
public static final String TASK_STARTUP_FAILURE_TOTAL = "task-startup-failure-total";
public static final String TASK_STARTUP_SUCCESS_PERCENTAGE = "task-startup-success-percentage";
public static final String TASK_STARTUP_SUCCESS_TOTAL = "task-startup-success-total";
/*********************************************************** connect ***********************************************************/
public static final String CONNECTOR_TOTAL_TASK_COUNT = "connector-total-task-count";
public static final String CONNECTOR_RUNNING_TASK_COUNT = "connector-running-task-count";
public static final String CONNECTOR_PAUSED_TASK_COUNT = "connector-paused-task-count";
public static final String CONNECTOR_FAILED_TASK_COUNT = "connector-failed-task-count";
public static final String CONNECTOR_UNASSIGNED_TASK_COUNT = "connector-unassigned-task-count";
public static final String BATCH_SIZE_AVG = "batch-size-avg";
public static final String BATCH_SIZE_MAX = "batch-size-max";
public static final String OFFSET_COMMIT_AVG_TIME_MS = "offset-commit-avg-time-ms";
public static final String OFFSET_COMMIT_MAX_TIME_MS = "offset-commit-max-time-ms";
public static final String OFFSET_COMMIT_FAILURE_PERCENTAGE = "offset-commit-failure-percentage";
public static final String OFFSET_COMMIT_SUCCESS_PERCENTAGE = "offset-commit-success-percentage";
public static final String POLL_BATCH_AVG_TIME_MS = "poll-batch-avg-time-ms";
public static final String POLL_BATCH_MAX_TIME_MS = "poll-batch-max-time-ms";
public static final String SOURCE_RECORD_ACTIVE_COUNT = "source-record-active-count";
public static final String SOURCE_RECORD_ACTIVE_COUNT_AVG = "source-record-active-count-avg";
public static final String SOURCE_RECORD_ACTIVE_COUNT_MAX = "source-record-active-count-max";
public static final String SOURCE_RECORD_POLL_RATE = "source-record-poll-rate";
public static final String SOURCE_RECORD_POLL_TOTAL = "source-record-poll-total";
public static final String SOURCE_RECORD_WRITE_RATE = "source-record-write-rate";
public static final String SOURCE_RECORD_WRITE_TOTAL = "source-record-write-total";
public static final String OFFSET_COMMIT_COMPLETION_RATE = "offset-commit-completion-rate";
public static final String OFFSET_COMMIT_COMPLETION_TOTAL = "offset-commit-completion-total";
public static final String OFFSET_COMMIT_SKIP_RATE = "offset-commit-skip-rate";
public static final String OFFSET_COMMIT_SKIP_TOTAL = "offset-commit-skip-total";
public static final String PARTITION_COUNT = "partition-count";
public static final String PUT_BATCH_AVG_TIME_MS = "put-batch-avg-time-ms";
public static final String PUT_BATCH_MAX_TIME_MS = "put-batch-max-time-ms";
public static final String SINK_RECORD_ACTIVE_COUNT = "sink-record-active-count";
public static final String SINK_RECORD_ACTIVE_COUNT_AVG = "sink-record-active-count-avg";
public static final String SINK_RECORD_ACTIVE_COUNT_MAX = "sink-record-active-count-max";
public static final String SINK_RECORD_LAG_MAX = "sink-record-lag-max";
public static final String SINK_RECORD_READ_RATE = "sink-record-read-rate";
public static final String SINK_RECORD_READ_TOTAL = "sink-record-read-total";
public static final String SINK_RECORD_SEND_RATE = "sink-record-send-rate";
public static final String SINK_RECORD_SEND_TOTAL = "sink-record-send-total";
public static final String DEADLETTERQUEUE_PRODUCE_FAILURES = "deadletterqueue-produce-failures";
public static final String DEADLETTERQUEUE_PRODUCE_REQUESTS = "deadletterqueue-produce-requests";
public static final String LAST_ERROR_TIMESTAMP = "last-error-timestamp";
public static final String TOTAL_ERRORS_LOGGED = "total-errors-logged";
public static final String TOTAL_RECORD_ERRORS = "total-record-errors";
public static final String TOTAL_RECORD_FAILURES = "total-record-failures";
public static final String TOTAL_RECORDS_SKIPPED = "total-records-skipped";
public static final String TOTAL_RETRIES = "total-retries";
private JmxAttribute() {
}
}

18
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxConnectorWrap.java
View File

@@ -28,9 +28,8 @@ import java.util.concurrent.atomic.AtomicInteger;
public class JmxConnectorWrap {
private static final Logger LOGGER = LoggerFactory.getLogger(JmxConnectorWrap.class);
private final Long physicalClusterId;
private final Integer brokerId;
//jmx打印日志时的附带信息
private final String clientLogIdent;
private final Long brokerStartupTime;
@@ -44,9 +43,8 @@ public class JmxConnectorWrap {
private JmxConfig jmxConfig;
public JmxConnectorWrap(Long physicalClusterId, Integer brokerId, Long brokerStartupTime, String host, Integer port, JmxConfig jmxConfig) {
this.physicalClusterId = physicalClusterId;
this.brokerId = brokerId;
public JmxConnectorWrap(String clientLogIdent, Long brokerStartupTime, String host, Integer port, JmxConfig jmxConfig) {
this.clientLogIdent=clientLogIdent;
this.brokerStartupTime = brokerStartupTime;
this.host = host;
@@ -93,7 +91,7 @@ public class JmxConnectorWrap {
jmxConnector = null;
} catch (IOException e) {
LOGGER.warn("close JmxConnector exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
LOGGER.warn("close JmxConnector exception, clientLogIdent:{} host:{} port:{}.", clientLogIdent, host, port, e);
}
}
@@ -176,12 +174,12 @@ public class JmxConnectorWrap {
}
jmxConnector = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), environment);
LOGGER.info("JMX connect success, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port);
LOGGER.info("JMX connect success, clientLogIdent:{} host:{} port:{}.", clientLogIdent, host, port);
return true;
} catch (MalformedURLException e) {
LOGGER.error("JMX url exception, physicalClusterId:{} brokerId:{} host:{} port:{} jmxUrl:{}", physicalClusterId, brokerId, host, port, jmxUrl, e);
LOGGER.error("JMX url exception, clientLogIdent:{} host:{} port:{} jmxUrl:{}", clientLogIdent, host, port, jmxUrl, e);
} catch (Exception e) {
LOGGER.error("JMX connect exception, physicalClusterId:{} brokerId:{} host:{} port:{}.", physicalClusterId, brokerId, host, port, e);
LOGGER.error("JMX connect exception, clientLogIdent:{} host:{} port:{}.", clientLogIdent, host, port, e);
}
return false;
}

14
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/jmx/JmxName.java
View File

@@ -69,6 +69,20 @@ public class JmxName {
public static final String JMX_ZK_SYNC_CONNECTS_PER_SEC = "kafka.server:type=SessionExpireListener,name=ZooKeeperSyncConnectsPerSec";
public static final String JMX_ZK_DISCONNECTORS_PER_SEC = "kafka.server:type=SessionExpireListener,name=ZooKeeperDisconnectsPerSec";
/*********************************************************** connect ***********************************************************/
public static final String JMX_CONNECT_WORKER_METRIC = "kafka.connect:type=connect-worker-metrics";
public static final String JMX_CONNECT_WORKER_CONNECTOR_METRIC = "kafka.connect:type=connect-worker-metrics,connector=%s";
public static final String JMX_CONNECTOR_TASK_CONNECTOR_METRIC = "kafka.connect:type=connector-task-metrics,connector=%s,task=%s";
public static final String JMX_CONNECTOR_SOURCE_TASK_METRICS = "kafka.connect:type=source-task-metrics,connector=%s,task=%s";
public static final String JMX_CONNECTOR_SINK_TASK_METRICS = "kafka.connect:type=sink-task-metrics,connector=%s,task=%s";
public static final String JMX_CONNECTOR_TASK_ERROR_METRICS = "kafka.connect:type=task-error-metrics,connector=%s,task=%s";
private JmxName() {
}
}

10
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/utils/CommonUtils.java
View File

@@ -7,6 +7,7 @@ import org.springframework.web.multipart.MultipartFile;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.ArrayList;
@@ -251,4 +252,13 @@ public class CommonUtils {
return true;
}
public static String getWorkerId(String url){
try {
URI uri = new URI(url);
return uri.getHost() + ":" + uri.getPort();
} catch (Exception e) {
return null;
}
}
}

Some files were not shown because too many files have changed in this diff Show More