mirror of
https://github.com/didi/KnowStreaming.git
synced 2026-01-04 11:52:07 +08:00
[Feature] 集群Group列表按照Group维度进行展示 (#580)
This commit is contained in:
zengqiao
@@ -1,11 +1,14 @@
|
||||
package com.xiaojukeji.know.streaming.km.biz.group;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterGroupSummaryDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetResetDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicPartitionKS;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupOverviewVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicConsumedDetailVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
@@ -22,6 +25,10 @@ public interface GroupManager {
|
||||
String searchGroupKeyword,
|
||||
PaginationBaseDTO dto);
|
||||
|
||||
PaginationResult<GroupTopicOverviewVO> pagingGroupTopicMembers(Long clusterPhyId, String groupName, PaginationBaseDTO dto);
|
||||
|
||||
PaginationResult<GroupOverviewVO> pagingClusterGroupsOverview(Long clusterPhyId, ClusterGroupSummaryDTO dto);
|
||||
|
||||
PaginationResult<GroupTopicConsumedDetailVO> pagingGroupTopicConsumedMetrics(Long clusterPhyId,
|
||||
String topicName,
|
||||
String groupName,
|
||||
@@ -31,4 +38,6 @@ public interface GroupManager {
|
||||
Result<Set<TopicPartitionKS>> listClusterPhyGroupPartitions(Long clusterPhyId, String groupName, Long startTime, Long endTime);
|
||||
|
||||
Result<Void> resetGroupOffsets(GroupOffsetResetDTO dto, String operator) throws Exception;
|
||||
|
||||
List<GroupTopicOverviewVO> getGroupTopicOverviewVOList (Long clusterPhyId, List<GroupMemberPO> groupMemberPOList);
|
||||
}
|
||||
|
||||
@@ -3,11 +3,14 @@ package com.xiaojukeji.know.streaming.km.biz.group.impl;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.biz.group.GroupManager;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.cluster.ClusterGroupSummaryDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.group.GroupOffsetResetDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.partition.PartitionOffsetDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.Group;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopicMember;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
@@ -15,11 +18,15 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicPartitionKS;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupOverviewVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicConsumedDetailVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.PaginationConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.GroupConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.OffsetTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.SortTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
@@ -71,30 +78,60 @@ public class GroupManagerImpl implements GroupManager {
|
||||
String searchGroupKeyword,
|
||||
PaginationBaseDTO dto) {
|
||||
PaginationResult<GroupMemberPO> paginationResult = groupService.pagingGroupMembers(clusterPhyId, topicName, groupName, searchTopicKeyword, searchGroupKeyword, dto);
|
||||
if (paginationResult.failed()) {
|
||||
return PaginationResult.buildFailure(paginationResult, dto);
|
||||
}
|
||||
|
||||
if (!paginationResult.hasData()) {
|
||||
return PaginationResult.buildSuc(new ArrayList<>(), paginationResult);
|
||||
}
|
||||
|
||||
// 获取指标
|
||||
Result<List<GroupMetrics>> metricsListResult = groupMetricService.listLatestMetricsAggByGroupTopicFromES(
|
||||
clusterPhyId,
|
||||
paginationResult.getData().getBizData().stream().map(elem -> new GroupTopic(elem.getGroupName(), elem.getTopicName())).collect(Collectors.toList()),
|
||||
Arrays.asList(GroupMetricVersionItems.GROUP_METRIC_LAG),
|
||||
AggTypeEnum.MAX
|
||||
);
|
||||
if (metricsListResult.failed()) {
|
||||
// 如果查询失败，则输出错误信息，但是依旧进行已有数据的返回
|
||||
log.error("method=pagingGroupMembers||clusterPhyId={}||topicName={}||groupName={}||result={}||errMsg=search es failed", clusterPhyId, topicName, groupName, metricsListResult);
|
||||
List<GroupTopicOverviewVO> groupTopicVOList = this.getGroupTopicOverviewVOList(clusterPhyId, paginationResult.getData().getBizData());
|
||||
|
||||
return PaginationResult.buildSuc(groupTopicVOList, paginationResult);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<GroupTopicOverviewVO> pagingGroupTopicMembers(Long clusterPhyId, String groupName, PaginationBaseDTO dto) {
|
||||
Group group = groupService.getGroupFromDB(clusterPhyId, groupName);
|
||||
|
||||
//没有topicMember则直接返回
|
||||
if (group == null || ValidateUtils.isEmptyList(group.getTopicMembers())) {
|
||||
return PaginationResult.buildSuc(dto);
|
||||
}
|
||||
|
||||
return PaginationResult.buildSuc(
|
||||
this.convert2GroupTopicOverviewVOList(paginationResult.getData().getBizData(), metricsListResult.getData()),
|
||||
paginationResult
|
||||
);
|
||||
//排序
|
||||
List<GroupTopicMember> groupTopicMembers = PaginationUtil.pageBySort(group.getTopicMembers(), PaginationConstant.DEFAULT_GROUP_TOPIC_SORTED_FIELD, SortTypeEnum.DESC.getSortType());
|
||||
|
||||
//分页
|
||||
PaginationResult<GroupTopicMember> paginationResult = PaginationUtil.pageBySubData(groupTopicMembers, dto);
|
||||
|
||||
List<GroupMemberPO> groupMemberPOList = paginationResult.getData().getBizData().stream().map(elem -> new GroupMemberPO(clusterPhyId, elem.getTopicName(), groupName, group.getState().getState(), elem.getMemberCount())).collect(Collectors.toList());
|
||||
|
||||
return PaginationResult.buildSuc(this.getGroupTopicOverviewVOList(clusterPhyId, groupMemberPOList), paginationResult);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<GroupOverviewVO> pagingClusterGroupsOverview(Long clusterPhyId, ClusterGroupSummaryDTO dto) {
|
||||
List<Group> groupList = groupService.listClusterGroups(clusterPhyId);
|
||||
|
||||
// 类型转化
|
||||
List<GroupOverviewVO> voList = groupList.stream().map(elem -> GroupConverter.convert2GroupOverviewVO(elem)).collect(Collectors.toList());
|
||||
|
||||
// 搜索groupName
|
||||
voList = PaginationUtil.pageByFuzzyFilter(voList, dto.getSearchGroupName(), Arrays.asList("name"));
|
||||
|
||||
//搜索topic
|
||||
if (!ValidateUtils.isBlank(dto.getSearchTopicName())) {
|
||||
voList = voList.stream().filter(elem -> {
|
||||
for (String topicName : elem.getTopicNameList()) {
|
||||
if (topicName.contains(dto.getSearchTopicName())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
// 分页 后 返回
|
||||
return PaginationUtil.pageBySubData(voList, dto);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -104,7 +141,7 @@ public class GroupManagerImpl implements GroupManager {
|
||||
List<String> latestMetricNames,
|
||||
PaginationSortDTO dto) throws NotExistException, AdminOperateException {
|
||||
// 获取消费组消费的TopicPartition列表
|
||||
Map<TopicPartition, Long> consumedOffsetMap = groupService.getGroupOffset(clusterPhyId, groupName);
|
||||
Map<TopicPartition, Long> consumedOffsetMap = groupService.getGroupOffsetFromKafka(clusterPhyId, groupName);
|
||||
List<Integer> partitionList = consumedOffsetMap.keySet()
|
||||
.stream()
|
||||
.filter(elem -> elem.topic().equals(topicName))
|
||||
@@ -113,7 +150,7 @@ public class GroupManagerImpl implements GroupManager {
|
||||
Collections.sort(partitionList);
|
||||
|
||||
// 获取消费组当前运行信息
|
||||
ConsumerGroupDescription groupDescription = groupService.getGroupDescription(clusterPhyId, groupName);
|
||||
ConsumerGroupDescription groupDescription = groupService.getGroupDescriptionFromKafka(clusterPhyId, groupName);
|
||||
|
||||
// 转换存储格式
|
||||
Map<TopicPartition, MemberDescription> tpMemberMap = new HashMap<>();
|
||||
@@ -166,13 +203,13 @@ public class GroupManagerImpl implements GroupManager {
|
||||
return rv;
|
||||
}
|
||||
|
||||
ConsumerGroupDescription description = groupService.getGroupDescription(dto.getClusterId(), dto.getGroupName());
|
||||
ConsumerGroupDescription description = groupService.getGroupDescriptionFromKafka(dto.getClusterId(), dto.getGroupName());
|
||||
if (ConsumerGroupState.DEAD.equals(description.state()) && !dto.isCreateIfNotExist()) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, "group不存在, 重置失败");
|
||||
}
|
||||
|
||||
if (!ConsumerGroupState.EMPTY.equals(description.state()) && !ConsumerGroupState.DEAD.equals(description.state())) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, String.format("group处于%s, 重置失败(仅Empty | Dead 情况可重置)", GroupStateEnum.getByRawState(description.state()).getState()));
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, String.format("group处于%s, 重置失败(仅Empty情况可重置)", GroupStateEnum.getByRawState(description.state()).getState()));
|
||||
}
|
||||
|
||||
// 获取offset
|
||||
@@ -185,6 +222,22 @@ public class GroupManagerImpl implements GroupManager {
|
||||
return groupService.resetGroupOffsets(dto.getClusterId(), dto.getGroupName(), offsetMapResult.getData(), operator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<GroupTopicOverviewVO> getGroupTopicOverviewVOList(Long clusterPhyId, List<GroupMemberPO> groupMemberPOList) {
|
||||
// 获取指标
|
||||
Result<List<GroupMetrics>> metricsListResult = groupMetricService.listLatestMetricsAggByGroupTopicFromES(
|
||||
clusterPhyId,
|
||||
groupMemberPOList.stream().map(elem -> new GroupTopic(elem.getGroupName(), elem.getTopicName())).collect(Collectors.toList()),
|
||||
Arrays.asList(GroupMetricVersionItems.GROUP_METRIC_LAG),
|
||||
AggTypeEnum.MAX
|
||||
);
|
||||
if (metricsListResult.failed()) {
|
||||
// 如果查询失败，则输出错误信息，但是依旧进行已有数据的返回
|
||||
log.error("method=completeMetricData||clusterPhyId={}||result={}||errMsg=search es failed", clusterPhyId, metricsListResult);
|
||||
}
|
||||
return this.convert2GroupTopicOverviewVOList(groupMemberPOList, metricsListResult.getData());
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
@@ -293,4 +346,31 @@ public class GroupManagerImpl implements GroupManager {
|
||||
);
|
||||
}
|
||||
|
||||
private List<GroupTopicOverviewVO> convert2GroupTopicOverviewVOList(String groupName, String state, List<GroupTopicMember> groupTopicList, List<GroupMetrics> metricsList) {
|
||||
if (metricsList == null) {
|
||||
metricsList = new ArrayList<>();
|
||||
}
|
||||
|
||||
// <TopicName, GroupMetrics>
|
||||
Map<String, GroupMetrics> metricsMap = new HashMap<>();
|
||||
for (GroupMetrics metrics : metricsList) {
|
||||
if (!groupName.equals(metrics.getGroup())) continue;
|
||||
metricsMap.put(metrics.getTopic(), metrics);
|
||||
}
|
||||
|
||||
List<GroupTopicOverviewVO> voList = new ArrayList<>();
|
||||
for (GroupTopicMember po : groupTopicList) {
|
||||
GroupTopicOverviewVO vo = ConvertUtil.obj2Obj(po, GroupTopicOverviewVO.class);
|
||||
vo.setGroupName(groupName);
|
||||
vo.setState(state);
|
||||
GroupMetrics metrics = metricsMap.get(po.getTopicName());
|
||||
if (metrics != null) {
|
||||
vo.setMaxLag(ConvertUtil.Float2Long(metrics.getMetrics().get(GroupMetricVersionItems.GROUP_METRIC_LAG)));
|
||||
}
|
||||
|
||||
voList.add(vo);
|
||||
}
|
||||
return voList;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package com.xiaojukeji.know.streaming.km.biz.topic;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicRecordDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicBrokersPartitionsSummaryVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicStateVO;
|
||||
@@ -23,4 +25,6 @@ public interface TopicStateManager {
|
||||
Result<List<TopicPartitionVO>> getTopicPartitions(Long clusterPhyId, String topicName, List<String> metricsNames);
|
||||
|
||||
Result<TopicBrokersPartitionsSummaryVO> getTopicBrokersPartitionsSummary(Long clusterPhyId, String topicName);
|
||||
|
||||
PaginationResult<GroupTopicOverviewVO> pagingTopicGroupsOverview(Long clusterPhyId, String topicName, String searchGroupName, PaginationBaseDTO dto);
|
||||
}
|
||||
|
||||
@@ -2,17 +2,22 @@ package com.xiaojukeji.know.streaming.km.biz.topic.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.biz.group.GroupManager;
|
||||
import com.xiaojukeji.know.streaming.km.biz.topic.TopicStateManager;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.topic.TopicRecordDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.broker.BrokerReplicaSummaryVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.group.GroupTopicOverviewVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicBrokersPartitionsSummaryVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicRecordVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.topic.TopicStateVO;
|
||||
@@ -32,6 +37,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.PaginationUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
|
||||
@@ -77,6 +83,12 @@ public class TopicStateManagerImpl implements TopicStateManager {
|
||||
@Autowired
|
||||
private TopicConfigService topicConfigService;
|
||||
|
||||
@Autowired
|
||||
private GroupService groupService;
|
||||
|
||||
@Autowired
|
||||
private GroupManager groupManager;
|
||||
|
||||
@Override
|
||||
public TopicBrokerAllVO getTopicBrokerAll(Long clusterPhyId, String topicName, String searchBrokerHost) throws NotExistException {
|
||||
Topic topic = topicService.getTopic(clusterPhyId, topicName);
|
||||
@@ -346,6 +358,19 @@ public class TopicStateManagerImpl implements TopicStateManager {
|
||||
return Result.buildSuc(vo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<GroupTopicOverviewVO> pagingTopicGroupsOverview(Long clusterPhyId, String topicName, String searchGroupName, PaginationBaseDTO dto) {
|
||||
PaginationResult<GroupMemberPO> paginationResult = groupService.pagingGroupMembers(clusterPhyId, topicName, "", "", searchGroupName, dto);
|
||||
|
||||
if (!paginationResult.hasData()) {
|
||||
return PaginationResult.buildSuc(new ArrayList<>(), paginationResult);
|
||||
}
|
||||
|
||||
List<GroupTopicOverviewVO> groupTopicVOList = groupManager.getGroupTopicOverviewVOList(clusterPhyId, paginationResult.getData().getBizData());
|
||||
|
||||
return PaginationResult.buildSuc(groupTopicVOList, paginationResult);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private boolean checkIfIgnore(ConsumerRecord<String, String> consumerRecord, String filterKey, String filterValue) {
|
||||
|
||||
Reference in New Issue
Block a user