8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 20:22:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

合并Master分支

Browse Source
This commit is contained in:
ZQKC
2023-07-07 13:09:28 +08:00
parent 9f6882cf0d 42195c3180
commit b1aa12bfa5
108 changed files with 2418 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
km-task/pom.xml
View File

@@ -5,13 +5,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-task</artifactId>
<version>${km.revision}</version>
<version>${revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<version>${revision}</version>
</parent>
<properties>

10
km-task/src/main/java/com/xiaojukeji/know/streaming/km/task/kafka/metadata/SyncConnectClusterAndWorkerTask.java
View File

@@ -18,8 +18,8 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupTypeEnum;
import com.xiaojukeji.know.streaming.km.common.enums.jmx.JmxEnum;
import com.xiaojukeji.know.streaming.km.core.service.connect.cluster.ConnectClusterService;
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerConnectorService;
import com.xiaojukeji.know.streaming.km.core.service.connect.worker.WorkerService;
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
import com.xiaojukeji.know.streaming.km.persistence.connect.cache.LoadedConnectClusterCache;
@@ -46,9 +46,6 @@ public class SyncConnectClusterAndWorkerTask extends AbstractAsyncMetadataDispat
@Autowired
private WorkerService workerService;
@Autowired
private WorkerConnectorService workerConnectorService;
@Autowired
private ConnectClusterService connectClusterService;
@@ -59,7 +56,6 @@ public class SyncConnectClusterAndWorkerTask extends AbstractAsyncMetadataDispat
//获取connect集群
List<Group> groupList = groupService.listClusterGroups(clusterPhy.getId()).stream().filter(elem->elem.getType()==GroupTypeEnum.CONNECT_CLUSTER).collect(Collectors.toList());
for (Group group: groupList) {
try {
KSGroupDescription ksGroupDescription = groupService.getGroupDescriptionFromKafka(clusterPhy, group.getName());
if (!ksGroupDescription.protocolType().equals(CONNECT_CLUSTER_PROTOCOL_TYPE)) {
@@ -104,7 +100,7 @@ public class SyncConnectClusterAndWorkerTask extends AbstractAsyncMetadataDispat
connectClusterId,
memberDescription.consumerId(),
memberDescription.host().substring(1),
Constant.INVALID_CODE,
JmxEnum.UNKNOWN.getPort(),
assignment.getWorkerState().url(),
assignment.getAssignment().leaderUrl(),
memberDescription.consumerId().equals(assignment.getAssignment().leader()) ? Constant.YES : Constant.NO
@@ -115,7 +111,7 @@ public class SyncConnectClusterAndWorkerTask extends AbstractAsyncMetadataDispat
connectClusterId,
memberDescription.consumerId(),
memberDescription.host().substring(1),
Constant.INVALID_CODE,
JmxEnum.UNKNOWN.getPort(),
"",
"",
Constant.NO

19
km-task/src/main/java/com/xiaojukeji/know/streaming/km/task/kafka/metadata/SyncKafkaGroupTask.java
View File

@@ -36,7 +36,7 @@ public class SyncKafkaGroupTask extends AbstractAsyncMetadataDispatchTask {
// 获取集群的Group列表
List<String> groupNameList = groupService.listGroupsFromKafka(clusterPhy);
TaskResult allSuccess = TaskResult.SUCCESS;
Set<String> getFailedGroupSet = new HashSet<>();
// 获取Group详细信息
List<Group> groupList = new ArrayList<>();
@@ -44,13 +44,16 @@ public class SyncKafkaGroupTask extends AbstractAsyncMetadataDispatchTask {
try {
Group group = groupService.getGroupFromKafka(clusterPhy, groupName);
if (group == null) {
// 获取到为空的 group 信息，直接忽略不要
continue;
}
groupList.add(group);
} catch (Exception e) {
log.error("method=processClusterTask||clusterPhyId={}||groupName={}||errMsg=exception", clusterPhy.getId(), groupName, e);
allSuccess = TaskResult.FAIL;
// 记录获取失败的 group 信息
getFailedGroupSet.add(groupName);
}
}
@@ -58,17 +61,9 @@ public class SyncKafkaGroupTask extends AbstractAsyncMetadataDispatchTask {
this.filterTopicIfTopicNotExist(clusterPhy.getId(), groupList);
// 更新DB中的Group信息
groupService.batchReplaceGroupsAndMembers(clusterPhy.getId(), groupList, triggerTimeUnitMs);
groupService.batchReplaceGroupsAndMembers(clusterPhy.getId(), groupList, getFailedGroupSet);
// 如果存在错误，则直接返回
if (!TaskResult.SUCCESS.equals(allSuccess)) {
return allSuccess;
}
// 删除历史的Group
groupService.deleteByUpdateTimeBeforeInDB(clusterPhy.getId(), new Date(triggerTimeUnitMs - 5 * 60 * 1000));
return allSuccess;
return getFailedGroupSet.isEmpty()? TaskResult.SUCCESS: TaskResult.FAIL;
}
private void filterTopicIfTopicNotExist(Long clusterPhyId, List<Group> groupList) {