8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2026-01-02 18:32:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Feature]HA-镜像Topic管理(#899)

Browse Source 
1、底层Kafka需要是滴滴版本的Kafka；
2、新增镜像Topic的增删改查；
3、新增镜像Topic的指标查看；
This commit is contained in:
zengqiao
2023-02-09 15:19:28 +08:00
committed by EricZeng
parent efdf624c67
commit 861faa5df5
25 changed files with 725 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/flusher/DatabaseDataFlusher.java
View File

@@ -3,6 +3,7 @@ package com.xiaojukeji.know.streaming.km.core.flusher;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
import com.xiaojukeji.know.streaming.km.common.bean.entity.ha.HaActiveStandbyRelation;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
@@ -13,6 +14,7 @@ import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
import com.xiaojukeji.know.streaming.km.persistence.cache.DataBaseDataLocalCache;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterMetricService;
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
import com.xiaojukeji.know.streaming.km.core.service.ha.HaActiveStandbyRelationService;
import com.xiaojukeji.know.streaming.km.core.service.health.checkresult.HealthCheckResultService;
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
@@ -50,6 +52,9 @@ public class DatabaseDataFlusher {
@Autowired
private PartitionService partitionService;
@Autowired
private HaActiveStandbyRelationService haActiveStandbyRelationService;
@PostConstruct
public void init() {
this.flushPartitionsCache();
@@ -59,6 +64,8 @@ public class DatabaseDataFlusher {
this.flushTopicLatestMetricsCache();
this.flushHealthCheckResultCache();
this.flushHaTopicCache();
}
@Scheduled(cron="0 0/1 * * * ?")
@@ -159,4 +166,12 @@ public class DatabaseDataFlusher {
});
}
}
@Scheduled(cron="0 0/1 * * * ?")
public void flushHaTopicCache() {
List<HaActiveStandbyRelation> haTopicList = haActiveStandbyRelationService.listAllTopicHa();
for (HaActiveStandbyRelation topic : haTopicList) {
DataBaseDataLocalCache.putHaTopic(topic.getStandbyClusterPhyId(), topic.getResName());
}
}
}

25
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/ha/HaActiveStandbyRelationService.java Normal file
View File

@@ -0,0 +1,25 @@
package com.xiaojukeji.know.streaming.km.core.service.ha;
import com.xiaojukeji.know.streaming.km.common.bean.entity.ha.HaActiveStandbyRelation;
import com.xiaojukeji.know.streaming.km.common.enums.ha.HaResTypeEnum;
import java.util.List;
public interface HaActiveStandbyRelationService {
/**
* 新增或者变更，支持幂等
*/
void batchReplaceTopicHA(Long activeClusterPhyId, Long standbyClusterPhyId, List<String> topicNameList);
/**
* 删除
*/
void batchDeleteTopicHA(Long activeClusterPhyId, Long standbyClusterPhyId, List<String> topicNameList);
/**
* 按照集群ID查询
*/
List<HaActiveStandbyRelation> listByClusterAndType(Long firstClusterId, HaResTypeEnum haResTypeEnum);
List<HaActiveStandbyRelation> listAllTopicHa();
}

106
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/ha/impl/HaActiveStandbyRelationServiceImpl.java Normal file
View File

@@ -0,0 +1,106 @@
package com.xiaojukeji.know.streaming.km.core.service.ha.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.xiaojukeji.know.streaming.km.common.bean.entity.ha.HaActiveStandbyRelation;
import com.xiaojukeji.know.streaming.km.common.bean.po.ha.HaActiveStandbyRelationPO;
import com.xiaojukeji.know.streaming.km.common.enums.ha.HaResTypeEnum;
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.core.service.ha.HaActiveStandbyRelationService;
import com.xiaojukeji.know.streaming.km.persistence.mysql.ha.HaActiveStandbyRelationDAO;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.xiaojukeji.know.streaming.km.common.enums.ha.HaResTypeEnum.MIRROR_TOPIC;
@Service
public class HaActiveStandbyRelationServiceImpl implements HaActiveStandbyRelationService {
@Autowired
private HaActiveStandbyRelationDAO haActiveStandbyRelationDAO;
@Override
public void batchReplaceTopicHA(Long activeClusterPhyId, Long standbyClusterPhyId, List<String> topicNameList) {
Map<String, HaActiveStandbyRelationPO> poMap = this.listPOs(activeClusterPhyId, standbyClusterPhyId, MIRROR_TOPIC)
.stream()
.collect(Collectors.toMap(HaActiveStandbyRelationPO::getResName, Function.identity()));
for (String topicName: topicNameList) {
HaActiveStandbyRelationPO oldPO = poMap.get(topicName);
if (oldPO != null) {
continue;
}
try {
haActiveStandbyRelationDAO.insert(new HaActiveStandbyRelationPO(activeClusterPhyId, standbyClusterPhyId, topicName, MIRROR_TOPIC.getCode()));
} catch (DuplicateKeyException dke) {
// ignore
}
}
}
@Override
public void batchDeleteTopicHA(Long activeClusterPhyId, Long standbyClusterPhyId, List<String> topicNameList) {
Map<String, HaActiveStandbyRelationPO> poMap = this.listPOs(activeClusterPhyId, standbyClusterPhyId, MIRROR_TOPIC)
.stream()
.collect(Collectors.toMap(HaActiveStandbyRelationPO::getResName, Function.identity()));
for (String topicName: topicNameList) {
HaActiveStandbyRelationPO oldPO = poMap.get(topicName);
if (oldPO == null) {
continue;
}
haActiveStandbyRelationDAO.deleteById(oldPO.getId());
}
}
@Override
public List<HaActiveStandbyRelation> listByClusterAndType(Long firstClusterId, HaResTypeEnum haResTypeEnum) {
// 查询HA列表
List<HaActiveStandbyRelationPO> poList = this.listPOs(firstClusterId, haResTypeEnum);
if (ValidateUtils.isNull(poList)) {
return new ArrayList<>();
}
return ConvertUtil.list2List(poList, HaActiveStandbyRelation.class);
}
@Override
public List<HaActiveStandbyRelation> listAllTopicHa() {
LambdaQueryWrapper<HaActiveStandbyRelationPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.eq(HaActiveStandbyRelationPO::getResType, MIRROR_TOPIC.getCode());
List<HaActiveStandbyRelationPO> poList = haActiveStandbyRelationDAO.selectList(lambdaQueryWrapper);
if (ValidateUtils.isNull(poList)) {
return new ArrayList<>();
}
return ConvertUtil.list2List(poList, HaActiveStandbyRelation.class);
}
private List<HaActiveStandbyRelationPO> listPOs(Long firstClusterId, HaResTypeEnum haResTypeEnum) {
LambdaQueryWrapper<HaActiveStandbyRelationPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.eq(HaActiveStandbyRelationPO::getResType, haResTypeEnum.getCode());
lambdaQueryWrapper.and(lambda ->
lambda.eq(HaActiveStandbyRelationPO::getActiveClusterPhyId, firstClusterId).or().eq(HaActiveStandbyRelationPO::getStandbyClusterPhyId, firstClusterId)
);
// 查询HA列表
return haActiveStandbyRelationDAO.selectList(lambdaQueryWrapper);
}
private List<HaActiveStandbyRelationPO> listPOs(Long activeClusterPhyId, Long standbyClusterPhyId, HaResTypeEnum haResTypeEnum) {
LambdaQueryWrapper<HaActiveStandbyRelationPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.eq(HaActiveStandbyRelationPO::getResType, haResTypeEnum.getCode());
lambdaQueryWrapper.eq(HaActiveStandbyRelationPO::getActiveClusterPhyId, activeClusterPhyId);
lambdaQueryWrapper.eq(HaActiveStandbyRelationPO::getStandbyClusterPhyId, standbyClusterPhyId);
// 查询HA列表
return haActiveStandbyRelationDAO.selectList(lambdaQueryWrapper);
}
}

25
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/OpTopicServiceImpl.java
View File

@@ -3,6 +3,7 @@ package com.xiaojukeji.know.streaming.km.core.service.topic.impl;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
import com.xiaojukeji.know.streaming.km.common.bean.entity.ha.HaActiveStandbyRelation;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicCreateParam;
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicParam;
@@ -12,11 +13,13 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
import com.xiaojukeji.know.streaming.km.common.converter.TopicConverter;
import com.xiaojukeji.know.streaming.km.common.enums.ha.HaResTypeEnum;
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
import com.xiaojukeji.know.streaming.km.core.service.ha.HaActiveStandbyRelationService;
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
import com.xiaojukeji.know.streaming.km.core.service.topic.OpTopicService;
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
@@ -70,6 +73,9 @@ public class OpTopicServiceImpl extends BaseKafkaVersionControlService implement
@Autowired
private KafkaZKDAO kafkaZKDAO;
@Autowired
private HaActiveStandbyRelationService haActiveStandbyRelationService;
@Override
protected VersionItemTypeEnum getVersionItemType() {
return SERVICE_OP_TOPIC;
@@ -138,6 +144,25 @@ public class OpTopicServiceImpl extends BaseKafkaVersionControlService implement
// 删除DB中的Topic数据
topicService.deleteTopicInDB(param.getClusterPhyId(), param.getTopicName());
//解除高可用Topic关联
List<HaActiveStandbyRelation> haActiveStandbyRelations = haActiveStandbyRelationService.listByClusterAndType(param.getClusterPhyId(), HaResTypeEnum.MIRROR_TOPIC);
for (HaActiveStandbyRelation activeStandbyRelation : haActiveStandbyRelations) {
if (activeStandbyRelation.getResName().equals(param.getTopicName())) {
try {
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(activeStandbyRelation.getStandbyClusterPhyId());
Properties haTopics = kafkaZkClient.getEntityConfigs("ha-topics", activeStandbyRelation.getResName());
if (haTopics.size() != 0) {
kafkaZkClient.setOrCreateEntityConfigs("ha-topics", activeStandbyRelation.getResName(), new Properties());
kafkaZkClient.createConfigChangeNotification("ha-topics/" + activeStandbyRelation.getResName());
}
haActiveStandbyRelationService.batchDeleteTopicHA(activeStandbyRelation.getActiveClusterPhyId(), activeStandbyRelation.getStandbyClusterPhyId(), Collections.singletonList(activeStandbyRelation.getResName()));
} catch (Exception e) {
log.error("method=deleteTopic||topicName:{}||errMsg=exception", activeStandbyRelation.getResName(), e);
return Result.buildFailure(e.getMessage());
}
}
}
// 记录操作
OplogDTO oplogDTO = new OplogDTO(operator,
OperationEnum.DELETE.getDesc(),

40
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/topic/impl/TopicMetricServiceImpl.java
View File

@@ -61,7 +61,7 @@ public class TopicMetricServiceImpl extends BaseMetricService implements TopicMe
public static final String TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_PARTITION_OF_BROKER_JMX = "getMetricFromKafkaByTotalPartitionOfBrokerJmx";
public static final String TOPIC_METHOD_GET_MESSAGES = "getMessages";
public static final String TOPIC_METHOD_GET_REPLICAS_COUNT = "getReplicasCount";
public static final String TOPIC_METHOD_GET_TOPIC_MIRROR_FETCH_LAG = "getTopicMirrorFetchLag";
@Autowired
private HealthStateService healthStateService;
@@ -98,6 +98,7 @@ public class TopicMetricServiceImpl extends BaseMetricService implements TopicMe
registerVCHandler( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_PARTITION_OF_BROKER_JMX, this::getMetricFromKafkaByTotalPartitionOfBrokerJmx );
registerVCHandler( TOPIC_METHOD_GET_REPLICAS_COUNT, this::getReplicasCount);
registerVCHandler( TOPIC_METHOD_GET_MESSAGES, this::getMessages);
registerVCHandler( TOPIC_METHOD_GET_TOPIC_MIRROR_FETCH_LAG, this::getTopicMirrorFetchLag);
}
@Override
@@ -502,4 +503,41 @@ public class TopicMetricServiceImpl extends BaseMetricService implements TopicMe
return aliveBrokerList.stream().filter(elem -> topic.getBrokerIdSet().contains(elem.getBrokerId())).collect(Collectors.toList());
}
private Result<List<TopicMetrics>> getTopicMirrorFetchLag(VersionItemParam param) {
TopicMetricParam topicMetricParam = (TopicMetricParam)param;
String topic = topicMetricParam.getTopic();
Long clusterId = topicMetricParam.getClusterId();
String metric = topicMetricParam.getMetric();
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
if (!DataBaseDataLocalCache.isHaTopic(clusterId, topic)) {
return Result.buildFailure(NOT_EXIST);
}
List<Broker> brokers = this.listAliveBrokersByTopic(clusterId, topic);
if(CollectionUtils.isEmpty(brokers)){return Result.buildFailure(BROKER_NOT_EXIST);}
Float sumLag = 0f;
for (Broker broker : brokers) {
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, broker.getBrokerId());
try {
String jmxObjectName = String.format(jmxInfo.getJmxObjectName(), topic);
Set<ObjectName> objectNameSet = jmxConnectorWrap.queryNames(new ObjectName(jmxObjectName), null);
for (ObjectName name : objectNameSet) {
Object attribute = jmxConnectorWrap.getAttribute(name, jmxInfo.getJmxAttribute());
sumLag += Float.valueOf(attribute.toString());
}
} catch (Exception e) {
LOGGER.error("method=getTopicMirrorFetchLag||cluster={}||brokerId={}||topic={}||metrics={}||jmx={}||msg={}",
clusterId, broker.getBrokerId(), topic, metric, jmxInfo.getJmxObjectName(), e.getClass().getName());
}
}
TopicMetrics topicMetric = new TopicMetrics(topic, clusterId, true);
topicMetric.putMetric(metric, sumLag);
return Result.buildSuc(Arrays.asList(topicMetric));
}
}

9
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/fe/FrontEndControlVersionItems.java
View File

@@ -33,6 +33,9 @@ public class FrontEndControlVersionItems extends BaseMetricVersionMetric {
private static final String FE_CREATE_TOPIC_CLEANUP_POLICY = "FECreateTopicCleanupPolicy";
private static final String FE_HA_CREATE_MIRROR_TOPIC = "FEHaCreateMirrorTopic";
private static final String FE_HA_DELETE_MIRROR_TOPIC = "FEHaDeleteMirrorTopic";
public FrontEndControlVersionItems(){}
@Override
@@ -80,6 +83,12 @@ public class FrontEndControlVersionItems extends BaseMetricVersionMetric {
itemList.add(buildItem().minVersion(VersionEnum.V_0_10_1_0).maxVersion(VersionEnum.V_MAX)
.name(FE_CREATE_TOPIC_CLEANUP_POLICY).desc("Topic-创建Topic-Cleanup-Policy"));
// HA-Topic复制
itemList.add(buildItem().minVersion(VersionEnum.V_2_5_0_D_300).maxVersion(VersionEnum.V_2_5_0_D_MAX)
.name(FE_HA_CREATE_MIRROR_TOPIC).desc("HA-创建Topic复制"));
itemList.add(buildItem().minVersion(VersionEnum.V_2_5_0_D_300).maxVersion(VersionEnum.V_2_5_0_D_MAX)
.name(FE_HA_DELETE_MIRROR_TOPIC).desc("HA-取消Topic复制"));
return itemList;
}
}

8
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/metrics/kafka/TopicMetricVersionItems.java
View File

@@ -2,6 +2,7 @@ package com.xiaojukeji.know.streaming.km.core.service.version.metrics.kafka;
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.BaseMetricVersionMetric;
import org.springframework.stereotype.Component;
@@ -36,6 +37,8 @@ public class TopicMetricVersionItems extends BaseMetricVersionMetric {
public static final String TOPIC_METRIC_BYTES_OUT_MIN_15 = "BytesOut_min_15";
public static final String TOPIC_METRIC_LOG_SIZE = "LogSize";
public static final String TOPIC_METRIC_UNDER_REPLICA_PARTITIONS = "PartitionURP";
public static final String TOPIC_METRIC_MIRROR_FETCH_LAG = "MirrorFetchLag";
public static final String TOPIC_METRIC_COLLECT_COST_TIME = Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME;
@Override
@@ -148,6 +151,11 @@ public class TopicMetricVersionItems extends BaseMetricVersionMetric {
.name(TOPIC_METRIC_COLLECT_COST_TIME).unit("秒").desc("采集Topic指标的耗时").category(CATEGORY_PERFORMANCE)
.extendMethod(TOPIC_METHOD_DO_NOTHING));
itemList.add(buildItem().minVersion(VersionEnum.V_2_5_0_D_300).maxVersion(VersionEnum.V_2_5_0_D_MAX)
.name(TOPIC_METRIC_MIRROR_FETCH_LAG).unit("条").desc("Topic复制延迟消息数").category(CATEGORY_FLOW)
.extend(buildJMXMethodExtend(TOPIC_METHOD_GET_TOPIC_MIRROR_FETCH_LAG)
.jmxObjectName(JMX_SERVER_TOPIC_MIRROR).jmxAttribute(VALUE)));
return itemList;
}
}