mirror of
https://github.com/didi/KnowStreaming.git
synced 2026-01-03 02:52:08 +08:00
初始化3.0.0版本
This commit is contained in:
zengqiao
91
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/cache/CollectMetricsLocalCache.java
vendored
Normal file
91
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/cache/CollectMetricsLocalCache.java
vendored
Normal file
@@ -0,0 +1,91 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.cache;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class CollectMetricsLocalCache {
|
||||
private static final Cache<String, Float> brokerMetricsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(60, TimeUnit.SECONDS)
|
||||
.maximumSize(2000)
|
||||
.build();
|
||||
|
||||
private static final Cache<String, List<TopicMetrics>> topicMetricsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(90, TimeUnit.SECONDS)
|
||||
.maximumSize(5000)
|
||||
.build();
|
||||
|
||||
private static final Cache<String, List<PartitionMetrics>> partitionMetricsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(90, TimeUnit.SECONDS)
|
||||
.maximumSize(10000)
|
||||
.build();
|
||||
|
||||
private static final Cache<String, Float> replicaMetricsValueCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(90, TimeUnit.SECONDS)
|
||||
.maximumSize(20000)
|
||||
.build();
|
||||
|
||||
public static Float getBrokerMetrics(Long clusterPhyId, Integer brokerId, String metricName) {
|
||||
return brokerMetricsCache.getIfPresent(CollectMetricsLocalCache.genBrokerMetricKey(clusterPhyId, brokerId, metricName));
|
||||
}
|
||||
|
||||
public static void putBrokerMetrics(Long clusterPhyId, Integer brokerId, String metricName, Float value) {
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
brokerMetricsCache.put(CollectMetricsLocalCache.genBrokerMetricKey(clusterPhyId, brokerId, metricName), value);
|
||||
}
|
||||
|
||||
public static List<TopicMetrics> getTopicMetrics(Long clusterPhyId, String topicName, String metricName) {
|
||||
return topicMetricsCache.getIfPresent(CollectMetricsLocalCache.genClusterTopicMetricKey(clusterPhyId, topicName, metricName));
|
||||
}
|
||||
|
||||
public static void putTopicMetrics(Long clusterPhyId, String topicName, String metricName, List<TopicMetrics> metricsList) {
|
||||
if (metricsList == null) {
|
||||
return;
|
||||
}
|
||||
topicMetricsCache.put(CollectMetricsLocalCache.genClusterTopicMetricKey(clusterPhyId, topicName, metricName), metricsList);
|
||||
}
|
||||
|
||||
public static List<PartitionMetrics> getPartitionMetricsList(Long clusterPhyId, String topicName, String metricName) {
|
||||
return partitionMetricsCache.getIfPresent(CollectMetricsLocalCache.genClusterTopicMetricKey(clusterPhyId, topicName, metricName));
|
||||
}
|
||||
|
||||
public static void putPartitionMetricsList(Long clusterPhyId, String topicName, String metricName, List<PartitionMetrics> metricsList) {
|
||||
if (metricsList == null) {
|
||||
return;
|
||||
}
|
||||
partitionMetricsCache.put(CollectMetricsLocalCache.genClusterTopicMetricKey(clusterPhyId, topicName, metricName), metricsList);
|
||||
}
|
||||
|
||||
public static Float getReplicaMetrics(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, String metricName) {
|
||||
return replicaMetricsValueCache.getIfPresent(CollectMetricsLocalCache.genReplicaMetricCacheKey(clusterPhyId, brokerId, topicName, partitionId, metricName));
|
||||
}
|
||||
|
||||
public static void putReplicaMetrics(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, String metricName, Float value) {
|
||||
if (value == null) {
|
||||
return;
|
||||
}
|
||||
replicaMetricsValueCache.put(CollectMetricsLocalCache.genReplicaMetricCacheKey(clusterPhyId, brokerId, topicName, partitionId, metricName), value);
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private static String genBrokerMetricKey(Long clusterPhyId, Integer brokerId, String metricName) {
|
||||
return clusterPhyId + "@" + brokerId + "@" + metricName;
|
||||
}
|
||||
|
||||
private static String genClusterTopicMetricKey(Long clusterPhyId, String topicName, String metricName) {
|
||||
return clusterPhyId + "@" + topicName + "@" + metricName;
|
||||
}
|
||||
|
||||
private static String genReplicaMetricCacheKey(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, String metricName) {
|
||||
return clusterPhyId + "@" + brokerId + "@" + topicName + "@" + partitionId + "@" + metricName;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.component.SpringTool;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterRunStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.flusher.zk.AbstractZKWatcher;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.AbstractClusterLoadedChangedHandler;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.NoArgsConstructor;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ZKWatcherManager extends AbstractClusterLoadedChangedHandler {
|
||||
protected static final ILog log = LogFactory.getLog(ZKWatcherManager.class);
|
||||
|
||||
private List<AbstractZKWatcher> watcherList;
|
||||
|
||||
@PostConstruct
|
||||
private void abstractInit() {
|
||||
watcherList = new ArrayList<>();
|
||||
watcherList.addAll(SpringTool.getBeansOfType(AbstractZKWatcher.class).values());
|
||||
}
|
||||
|
||||
@Scheduled(cron = "0 0/1 * * * ?")
|
||||
public void scheduledTriggerFlush() {
|
||||
for (ClusterPhy clusterPhy: LoadedClusterPhyCache.listAll().values()) {
|
||||
if (!clusterPhy.getRunState().equals(ClusterRunStateEnum.RUN_ZK.getRunState())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (AbstractZKWatcher abstractZKWatcher: watcherList) {
|
||||
try {
|
||||
FutureUtil.quickStartupFutureUtil.submitTask(
|
||||
() -> {
|
||||
log.debug("class={}||method=scheduledTriggerFlush||clusterPhyId={}||msg=flush task start"
|
||||
, abstractZKWatcher.getClass().getSimpleName(), clusterPhy.getId());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
abstractZKWatcher.flush(clusterPhy);
|
||||
|
||||
log.info("class={}||method=scheduledTriggerFlush||clusterPhyId={}||costTime={}ms||msg=flush task finished"
|
||||
, abstractZKWatcher.getClass().getSimpleName(), clusterPhy.getId(), System.currentTimeMillis() - startTime);
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("method=scheduledTriggerFlush||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void add(ClusterPhy clusterPhy) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void remove(ClusterPhy clusterPhy) {
|
||||
for (AbstractZKWatcher abstractMetadataFlush: watcherList) {
|
||||
abstractMetadataFlush.remove(clusterPhy.getId());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void modify(ClusterPhy newClusterPhy, ClusterPhy oldClusterPhy) {
|
||||
if (newClusterPhy.getZookeeper().equals(oldClusterPhy.getZookeeper())
|
||||
&& newClusterPhy.getBootstrapServers().equals(oldClusterPhy.getBootstrapServers())
|
||||
&& newClusterPhy.getClientProperties().equals(oldClusterPhy.getClientProperties())
|
||||
&& newClusterPhy.getRunState().equals(oldClusterPhy.getRunState())) {
|
||||
// 集群ZK及服务地址等信息无变化，则元信息无需变化，直接返回
|
||||
return;
|
||||
}
|
||||
|
||||
this.remove(newClusterPhy);
|
||||
|
||||
this.add(newClusterPhy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CopyOnWriteArraySet;
|
||||
|
||||
public abstract class AbstractZKWatcher {
|
||||
private static final ILog log = LogFactory.getLog(AbstractZKWatcher.class);
|
||||
|
||||
@Autowired
|
||||
protected KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Autowired
|
||||
protected KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
private final Set<Long> watchedClusterPhyId = new CopyOnWriteArraySet<>();
|
||||
|
||||
protected volatile Long recordedZKClientCreateTime;
|
||||
|
||||
protected abstract boolean addWatch(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient);
|
||||
|
||||
protected abstract void handleZKClientNotExist(ClusterPhy clusterPhy);
|
||||
|
||||
/**
|
||||
* 检查并触发任务执行
|
||||
* @param clusterPhy 物理集群信息
|
||||
*/
|
||||
public void flush(ClusterPhy clusterPhy) {
|
||||
Long newZKClientCreateTime = kafkaAdminZKClient.getZKClientCreateTime(clusterPhy.getId());
|
||||
if (newZKClientCreateTime == null || !newZKClientCreateTime.equals(recordedZKClientCreateTime)) {
|
||||
// 如果ZK客户端发生过变化，则移除已经watched的集群
|
||||
watchedClusterPhyId.remove(clusterPhy.getId());
|
||||
}
|
||||
|
||||
if (watchedClusterPhyId.contains(clusterPhy.getId())) {
|
||||
// 已经加载
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.addWatch(clusterPhy, kafkaAdminZKClient.getClient(clusterPhy.getId()))) {
|
||||
watchedClusterPhyId.add(clusterPhy.getId());
|
||||
recordedZKClientCreateTime = newZKClientCreateTime;
|
||||
}
|
||||
} catch (NotExistException nee) {
|
||||
this.handleZKClientNotExist(clusterPhy);
|
||||
} catch (Exception e) {
|
||||
log.error("method=flush||clusterPhy={}||errMsg=exception.", clusterPhy, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除缓存
|
||||
* @param clusterPhyId 物理集群ID
|
||||
*/
|
||||
public void remove(Long clusterPhyId) {
|
||||
watchedClusterPhyId.remove(clusterPhyId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.core.flusher.zk.handler.BrokersNodeChangeHandler;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import kafka.zk.BrokerIdsZNode;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class BrokersNodeWatcher extends AbstractZKWatcher {
|
||||
private static final ILog log = LogFactory.getLog(BrokersNodeWatcher.class);
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private KafkaChangeRecordService kafkaChangeRecordService;
|
||||
|
||||
@Override
|
||||
protected boolean addWatch(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
return this.loadMetadataByWatchZK(clusterPhy, kafkaZkClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleZKClientNotExist(ClusterPhy clusterPhy) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private boolean loadMetadataByWatchZK(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
try {
|
||||
if (kafkaZkClient.currentZooKeeper().exists(BrokerIdsZNode.path(), false) == null) {
|
||||
log.info("ignore add physicalCluster, zk path={} not exist, physicalClusterName:{}.", BrokerIdsZNode.path(), clusterPhy.getName());
|
||||
return false;
|
||||
}
|
||||
|
||||
//增加Broker监控
|
||||
BrokersNodeChangeHandler brokerChangeHandler = new BrokersNodeChangeHandler(clusterPhy.getId(), kafkaZKDAO, kafkaChangeRecordService, brokerService);
|
||||
kafkaAdminZKClient.getClient(clusterPhy.getId()).registerZNodeChildChangeHandler(brokerChangeHandler);
|
||||
brokerChangeHandler.init();
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
log.error("load physicalCluster failed, clusterPhy:{}.", clusterPhy, e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.core.flusher.zk.handler.ConfigNotificationNodeChangeHandler;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import kafka.zk.TopicsZNode;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
@Component
|
||||
public class ConfigChangeNodeWatcher extends AbstractZKWatcher {
|
||||
private static final ILog logger = LogFactory.getLog(ConfigChangeNodeWatcher.class);
|
||||
|
||||
@Autowired
|
||||
private KafkaChangeRecordService kafkaChangeRecordService;
|
||||
|
||||
@Override
|
||||
protected boolean addWatch(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
return this.loadChangeNotificationByWatchZK(clusterPhy, kafkaZkClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleZKClientNotExist(ClusterPhy clusterPhy) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private boolean loadChangeNotificationByWatchZK(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
try {
|
||||
if (kafkaZkClient.currentZooKeeper().exists(TopicsZNode.path(), false) == null) {
|
||||
logger.info("ignore add physicalCluster, zk path={} not exist, physicalClusterName:{}.", TopicsZNode.path(), clusterPhy.getName());
|
||||
return false;
|
||||
}
|
||||
|
||||
//增加config-change监控
|
||||
ConfigNotificationNodeChangeHandler changeHandler = new ConfigNotificationNodeChangeHandler(clusterPhy.getId(), kafkaZKDAO, kafkaChangeRecordService);
|
||||
kafkaAdminZKClient.getClient(clusterPhy.getId()).registerZNodeChildChangeHandler(changeHandler);
|
||||
changeHandler.init();
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
logger.error("method=loadChangeNotificationByWatchZK||clusterPhyPO={}||errMsg=exception!", clusterPhy, e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.flusher.zk.handler.ControllerNodeChangeHandler;
|
||||
import kafka.zk.ControllerZNode;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
@Component
|
||||
public class ControllerNodeWatcher extends AbstractZKWatcher {
|
||||
private static final ILog log = LogFactory.getLog(ControllerNodeWatcher.class);
|
||||
|
||||
@Autowired
|
||||
private KafkaControllerService kafkaControllerService;
|
||||
|
||||
@Autowired
|
||||
private KafkaChangeRecordService kafkaChangeRecordService;
|
||||
|
||||
@Override
|
||||
protected boolean addWatch(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
return this.loadMetadataByWatchZK(clusterPhy, kafkaZkClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleZKClientNotExist(ClusterPhy clusterPhy) {
|
||||
kafkaControllerService.setNoKafkaController(clusterPhy.getId(), System.currentTimeMillis() / 1000L * 1000L);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private boolean loadMetadataByWatchZK(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
try {
|
||||
if (kafkaZkClient.currentZooKeeper().exists(ControllerZNode.path(), false) == null) {
|
||||
log.info("ignore add physicalCluster, zk path={} not exist, physicalClusterName:{}.", ControllerZNode.path(), clusterPhy.getName());
|
||||
return false;
|
||||
}
|
||||
|
||||
ControllerNodeChangeHandler changeHandler = new ControllerNodeChangeHandler(clusterPhy.getId(), kafkaZKDAO, kafkaChangeRecordService, kafkaControllerService);
|
||||
changeHandler.init();
|
||||
kafkaAdminZKClient.getClient(clusterPhy.getId()).registerZNodeChangeHandlerAndCheckExistence(changeHandler);
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
log.error("load physicalCluster failed, clusterPhy:{}.", clusterPhy, e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.core.flusher.zk.handler.TopicsNodeChangeHandler;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import kafka.zk.TopicsZNode;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
@Component
|
||||
public class TopicsNodeWatcher extends AbstractZKWatcher {
|
||||
private static final ILog logger = LogFactory.getLog(TopicsNodeWatcher.class);
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private KafkaChangeRecordService kafkaChangeRecordService;
|
||||
|
||||
@Override
|
||||
protected boolean addWatch(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
return this.loadMetadataByWatchZK(clusterPhy, kafkaZkClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleZKClientNotExist(ClusterPhy clusterPhy) {
|
||||
// 无需对ZK客户端不存在时进行处理
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private boolean loadMetadataByWatchZK(ClusterPhy clusterPhy, KafkaZkClient kafkaZkClient) {
|
||||
try {
|
||||
if (kafkaZkClient.currentZooKeeper().exists(TopicsZNode.path(), false) == null) {
|
||||
logger.info("ignore add physicalCluster, zk path={} not exist, physicalClusterName:{}.", TopicsZNode.path(), clusterPhy.getName());
|
||||
return false;
|
||||
}
|
||||
|
||||
//增加Topic监控
|
||||
TopicsNodeChangeHandler changeHandler = new TopicsNodeChangeHandler(clusterPhy.getId(), kafkaZKDAO, kafkaChangeRecordService, topicService);
|
||||
kafkaAdminZKClient.getClient(clusterPhy.getId()).registerZNodeChildChangeHandler(changeHandler);
|
||||
changeHandler.init();
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
logger.error("load physicalCluster failed, clusterPhy:{}.", clusterPhy, e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk.handler;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
|
||||
|
||||
public abstract class AbstractZKHandler {
|
||||
private static final ILog log = LogFactory.getLog(AbstractZKHandler.class);
|
||||
|
||||
protected Long clusterPhyId;
|
||||
|
||||
protected KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
protected KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
protected KafkaChangeRecordService kafkaChangeRecordService;
|
||||
|
||||
public abstract void init();
|
||||
|
||||
public void reInitDataIfException() {
|
||||
// 回退2秒，会阻塞处理线程，因此回退时间不可设置太长
|
||||
BackoffUtils.backoff(2000);
|
||||
|
||||
if (!LoadedClusterPhyCache.containsByPhyId(this.clusterPhyId)) {
|
||||
// 集群不存在，则直接忽略重试
|
||||
log.info("method=reInitDataIfException||clusterPhyId={}||msg=not exist", this.clusterPhyId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
kafkaAdminZKClient.getClient(clusterPhyId);
|
||||
} catch (NotExistException nee) {
|
||||
// 客户端不存在，则也忽略重试
|
||||
log.info("method=reInitDataIfException||clusterPhyId={}||errMsg=exception", this.clusterPhyId, nee);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
init();
|
||||
}
|
||||
|
||||
protected AbstractZKHandler(Long clusterPhyId, KafkaZKDAO kafkaZKDAO, KafkaChangeRecordService kafkaChangeRecordService) {
|
||||
this.clusterPhyId = clusterPhyId;
|
||||
this.kafkaZKDAO = kafkaZKDAO;
|
||||
this.kafkaChangeRecordService = kafkaChangeRecordService;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk.handler;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.changerecord.KafkaChangeRecordPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.BrokerIdsZNode;
|
||||
import kafka.zookeeper.ZNodeChildChangeHandler;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
public class BrokersNodeChangeHandler extends AbstractZKHandler implements ZNodeChildChangeHandler {
|
||||
private static final ILog log = LogFactory.getLog(BrokersNodeChangeHandler.class);
|
||||
|
||||
private final BrokerService brokerService;
|
||||
|
||||
private static final FutureUtil<Void> futureUtil = FutureUtil.init(
|
||||
"BrokerChangeHandlerThread",
|
||||
1,
|
||||
1,
|
||||
Integer.MAX_VALUE
|
||||
);
|
||||
|
||||
public BrokersNodeChangeHandler(Long clusterPhyId, KafkaZKDAO kafkaZKDAO, KafkaChangeRecordService kafkaChangeRecordService, BrokerService brokerService) {
|
||||
super(clusterPhyId, kafkaZKDAO, kafkaChangeRecordService);
|
||||
this.brokerService = brokerService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.handleChildChange();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String path() {
|
||||
return BrokerIdsZNode.path();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleChildChange() {
|
||||
long triggerTime = System.currentTimeMillis() / 1000L * 1000L;
|
||||
|
||||
List<Broker> brokerList = new ArrayList<>();
|
||||
try {
|
||||
for (String brokerId: kafkaZKDAO.getChildren(clusterPhyId, BrokerIdsZNode.path(), true)) {
|
||||
brokerList.add(kafkaZKDAO.getBrokerMetadata(clusterPhyId, Integer.valueOf(brokerId)));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=handleChildChange||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
|
||||
// 出现异常时，进行重试
|
||||
reInitDataIfException();
|
||||
return;
|
||||
}
|
||||
|
||||
// 记录变更
|
||||
futureUtil.submitTask(
|
||||
() -> recordChangeToDB(this.clusterPhyId, brokerList, triggerTime)
|
||||
);
|
||||
|
||||
// 更新DB
|
||||
futureUtil.submitTask(
|
||||
() -> updateDBData(this.clusterPhyId, brokerList)
|
||||
);
|
||||
}
|
||||
|
||||
private void recordChangeToDB(Long clusterPhyId, List<Broker> zkBrokerList, long triggerTime) {
|
||||
try {
|
||||
// DB中记录的
|
||||
Map<Integer, Broker> dbBrokerMap = brokerService.listAllBrokersFromDB(clusterPhyId).stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
|
||||
|
||||
for (Broker zkBroker: zkBrokerList) {
|
||||
Broker dbBroker = dbBrokerMap.remove(zkBroker.getBrokerId());
|
||||
if (dbBroker == null) {
|
||||
// 这是新增的Broker
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_BROKER,
|
||||
String.valueOf(zkBroker.getBrokerId()),
|
||||
OperationEnum.ADD,
|
||||
new Date(zkBroker.getStartTimestamp())
|
||||
));
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!zkBroker.getStartTimestamp().equals(dbBroker.getStartTimestamp())) {
|
||||
// 这是修改的Broker
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_BROKER,
|
||||
String.valueOf(zkBroker.getBrokerId()),
|
||||
OperationEnum.EDIT,
|
||||
new Date(zkBroker.getStartTimestamp())
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// 这是下线的Broker
|
||||
for (Broker dbBroker: dbBrokerMap.values()) {
|
||||
if (!dbBroker.alive()) {
|
||||
// broker本身就未存活，则直接跳过
|
||||
continue;
|
||||
}
|
||||
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_BROKER,
|
||||
String.valueOf(dbBroker.getBrokerId()),
|
||||
OperationEnum.DELETE,
|
||||
new Date(triggerTime),
|
||||
new Date(dbBroker.getStartTimestamp()) // 用下线Broker的启动时间，避免重复记录删除动作
|
||||
));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=recordChangeToDB||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateDBData(Long clusterPhyId, List<Broker> zkBrokerList) {
|
||||
try {
|
||||
brokerService.updateAliveBrokers(clusterPhyId, zkBrokerList);
|
||||
} catch (Exception e) {
|
||||
log.error("method=updateDBData||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk.handler;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.changerecord.KafkaChangeRecordPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.KafkaConfigTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationBaseData;
|
||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV1;
|
||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.config.ConfigChangeNotificationDataV2;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.ConfigEntityChangeNotificationZNode;
|
||||
import kafka.zookeeper.ZNodeChildChangeHandler;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public class ConfigNotificationNodeChangeHandler extends AbstractZKHandler implements ZNodeChildChangeHandler {
|
||||
private static final ILog log = LogFactory.getLog(ConfigNotificationNodeChangeHandler.class);
|
||||
|
||||
private static final FutureUtil<Void> futureUtil = FutureUtil.init(
|
||||
"ConfigNotificationHandlerThread",
|
||||
1,
|
||||
1,
|
||||
Integer.MAX_VALUE
|
||||
);
|
||||
|
||||
private volatile String startConfigChangeNode = "config_change_";
|
||||
|
||||
public ConfigNotificationNodeChangeHandler(Long clusterPhyId, KafkaZKDAO kafkaZKDAO, KafkaChangeRecordService kafkaChangeRecordService) {
|
||||
super(clusterPhyId, kafkaZKDAO, kafkaChangeRecordService);
|
||||
this.kafkaChangeRecordService = kafkaChangeRecordService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.handleChildChange();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String path() {
|
||||
return ConfigEntityChangeNotificationZNode.path();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleChildChange() {
|
||||
List<String> notificationList = new ArrayList<>();
|
||||
try {
|
||||
notificationList.addAll(kafkaZKDAO.getChildren(clusterPhyId, this.path(), true));
|
||||
} catch (Exception e) {
|
||||
log.error("method=handleChildChange||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
|
||||
// 出现异常时，进行重试
|
||||
reInitDataIfException();
|
||||
return;
|
||||
}
|
||||
|
||||
// 记录变更
|
||||
futureUtil.submitTask(
|
||||
() -> recordChangeToDB(this.clusterPhyId, notificationList)
|
||||
);
|
||||
}
|
||||
|
||||
private void recordChangeToDB(Long clusterPhyId, List<String> notificationList) {
|
||||
try {
|
||||
boolean success = true;
|
||||
for (String notification: notificationList) {
|
||||
try {
|
||||
if (startConfigChangeNode.compareTo(notification) >= 0) {
|
||||
// 已经处理过的，则不进行处理
|
||||
continue;
|
||||
}
|
||||
|
||||
this.recordChangeToDB(this.clusterPhyId, notification);
|
||||
|
||||
if (success) {
|
||||
startConfigChangeNode = notification;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=recordChangeToDB||clusterPhyId={}||notification={}||errMsg=exception!", clusterPhyId, notification, e);
|
||||
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=recordChangeToDB||clusterPhyId={}||errMsg=exception!", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void recordChangeToDB(Long clusterPhyId, String notification) throws Exception {
|
||||
String changeZNode = this.path() + "/" + notification;
|
||||
|
||||
Tuple<byte[], Stat> dataTuple = kafkaZKDAO.getDataAndStat(clusterPhyId, changeZNode);
|
||||
if (System.currentTimeMillis() - dataTuple.getV2().getCtime() >= (10L * 60L * 1000L)) {
|
||||
// 忽略历史的
|
||||
return;
|
||||
}
|
||||
|
||||
KafkaChangeRecordPO recordPO = null;
|
||||
|
||||
ConfigChangeNotificationBaseData baseData = JSONObject.parseObject(dataTuple.getV1(), ConfigChangeNotificationBaseData.class);
|
||||
if (ConfigChangeNotificationDataV1.CHANGE_DATA_VERSION.equals(baseData.getVersion())) {
|
||||
ConfigChangeNotificationDataV1 dataV1 = JSONObject.parseObject(dataTuple.getV1(), ConfigChangeNotificationDataV1.class);
|
||||
|
||||
recordPO = new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
KafkaConfigTypeEnum.getByConfigName(dataV1.getEntityType()),
|
||||
dataV1.getEntityName(),
|
||||
OperationEnum.EDIT,
|
||||
new Date(dataTuple.getV2().getCtime())
|
||||
);
|
||||
|
||||
|
||||
} else if (ConfigChangeNotificationDataV2.CHANGE_DATA_VERSION.equals(baseData.getVersion())) {
|
||||
ConfigChangeNotificationDataV2 dataV2 = JSONObject.parseObject(dataTuple.getV1(), ConfigChangeNotificationDataV2.class);
|
||||
|
||||
int idx = dataV2.getEntityPath().indexOf("/");
|
||||
if (idx < 0) {
|
||||
log.error("method=recordChangeToDB||clusterPhyId={}||notification={}||notifyData={}||errMsg=data illegal.", clusterPhyId, notification, dataV2);
|
||||
return;
|
||||
}
|
||||
|
||||
recordPO = new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
KafkaConfigTypeEnum.getByConfigName(dataV2.getEntityPath().substring(0, idx)),
|
||||
dataV2.getEntityPath().substring(idx + 1),
|
||||
OperationEnum.EDIT,
|
||||
new Date(dataTuple.getV2().getCtime())
|
||||
);
|
||||
}
|
||||
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(recordPO);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk.handler;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.changerecord.KafkaChangeRecordPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BackoffUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.ControllerZNode;
|
||||
import kafka.zookeeper.ZNodeChangeHandler;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
public class ControllerNodeChangeHandler extends AbstractZKHandler implements ZNodeChangeHandler {
|
||||
private static final ILog log = LogFactory.getLog(ControllerNodeChangeHandler.class);
|
||||
|
||||
private static final FutureUtil<Void> futureUtil = FutureUtil.init(
|
||||
"ControllerChangeHandlerThread",
|
||||
1,
|
||||
1,
|
||||
Integer.MAX_VALUE
|
||||
);
|
||||
|
||||
private final KafkaControllerService kafkaControllerService;
|
||||
|
||||
public ControllerNodeChangeHandler(Long clusterPhyId, KafkaZKDAO kafkaZKDAO, KafkaChangeRecordService kafkaChangeRecordService, KafkaControllerService kafkaControllerService) {
|
||||
super(clusterPhyId, kafkaZKDAO, kafkaChangeRecordService);
|
||||
this.kafkaControllerService = kafkaControllerService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.handleChange();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String path() {
|
||||
return ControllerZNode.path();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleCreation() {
|
||||
this.handleChange();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleDeletion() {
|
||||
this.handleChange();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleDataChange() {
|
||||
this.handleChange();
|
||||
}
|
||||
|
||||
public void handleChange() {
|
||||
long triggerTime = System.currentTimeMillis() / 1000L * 1000L;
|
||||
|
||||
// 回退等待3秒, 绝大多数情况下，3秒都可以选出新的controller，如果3秒后还没有，则记录该3秒无controller的情况
|
||||
BackoffUtils.backoff(3000);
|
||||
|
||||
// 记录变更
|
||||
futureUtil.submitTask(
|
||||
() -> recordChangeToDB(this.clusterPhyId, triggerTime)
|
||||
);
|
||||
|
||||
// 更新DB
|
||||
futureUtil.submitTask(
|
||||
() -> updateDBData(this.clusterPhyId, triggerTime)
|
||||
);
|
||||
}
|
||||
|
||||
private void recordChangeToDB(Long clusterPhyId, long triggerTime) {
|
||||
try {
|
||||
KafkaController kafkaController = kafkaZKDAO.getKafkaController(clusterPhyId, true);
|
||||
|
||||
// 记录变更
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_CONTROLLER,
|
||||
String.valueOf(kafkaController != null ? kafkaController.getBrokerId(): Constant.INVALID_CODE),
|
||||
kafkaController != null ? OperationEnum.SWITCH: OperationEnum.DELETE,
|
||||
kafkaController != null ? new Date(kafkaController.getTimestamp()) : new Date(triggerTime)
|
||||
));
|
||||
} catch (Exception e) {
|
||||
log.error("method=recordChangeToDB||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateDBData(Long clusterPhyId, long triggerTime) {
|
||||
try {
|
||||
KafkaController kafkaController = kafkaZKDAO.getKafkaController(clusterPhyId, true);
|
||||
if (kafkaController == null) {
|
||||
kafkaControllerService.setNoKafkaController(clusterPhyId, triggerTime);
|
||||
} else {
|
||||
kafkaControllerService.insertAndIgnoreDuplicateException(kafkaController);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=updateDBData||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
|
||||
// 出现异常时，进行重试
|
||||
reInitDataIfException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.flusher.zk.handler;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.changerecord.KafkaChangeRecordPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.FutureUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.TopicsZNode;
|
||||
import kafka.zookeeper.ZNodeChildChangeHandler;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
public class TopicsNodeChangeHandler extends AbstractZKHandler implements ZNodeChildChangeHandler {
|
||||
private static final ILog log = LogFactory.getLog(TopicsNodeChangeHandler.class);
|
||||
|
||||
private final TopicService topicService;
|
||||
|
||||
private static final FutureUtil<Void> futureUtil = FutureUtil.init(
|
||||
"TopicChangeHandlerThread",
|
||||
1,
|
||||
1,
|
||||
Integer.MAX_VALUE
|
||||
);
|
||||
|
||||
public TopicsNodeChangeHandler(Long clusterPhyId, KafkaZKDAO kafkaZKDAO, KafkaChangeRecordService kafkaChangeRecordService, TopicService topicService) {
|
||||
super(clusterPhyId, kafkaZKDAO, kafkaChangeRecordService);
|
||||
this.topicService = topicService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.handleChildChange();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String path() {
|
||||
return TopicsZNode.path();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleChildChange() {
|
||||
long triggerTime = System.currentTimeMillis() / 1000L * 1000L;
|
||||
|
||||
List<Topic> topicList = new ArrayList<>();
|
||||
try {
|
||||
for (String topicName: kafkaZKDAO.getChildren(clusterPhyId, TopicsZNode.path(), true)) {
|
||||
topicList.add(kafkaZKDAO.getTopicMetadata(clusterPhyId, topicName));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=handleChildChange||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
|
||||
// 出现异常时，进行重试
|
||||
reInitDataIfException();
|
||||
return;
|
||||
}
|
||||
|
||||
// 记录变更
|
||||
futureUtil.submitTask(
|
||||
() -> recordChangeToDB(this.clusterPhyId, topicList, triggerTime)
|
||||
);
|
||||
|
||||
// 更新DB
|
||||
futureUtil.submitTask(
|
||||
() -> updateDBData(this.clusterPhyId, topicList)
|
||||
);
|
||||
}
|
||||
|
||||
private void recordChangeToDB(Long clusterPhyId, List<Topic> zkTopicList, long triggerTime) {
|
||||
try {
|
||||
// DB中记录的
|
||||
Map<String, Topic> dbTopicMap = topicService.listTopicsFromDB(clusterPhyId).stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity()));
|
||||
|
||||
for (Topic zkTopic: zkTopicList) {
|
||||
Topic dbTopic = dbTopicMap.remove(zkTopic.getTopicName());
|
||||
if (dbTopic == null) {
|
||||
// 这是新增的Topic
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_TOPIC,
|
||||
zkTopic.getTopicName(),
|
||||
OperationEnum.ADD,
|
||||
new Date(zkTopic.getCreateTime())
|
||||
));
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!zkTopic.getUpdateTime().equals(dbTopic.getUpdateTime())) {
|
||||
// 这是修改的Topic
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_TOPIC,
|
||||
zkTopic.getTopicName(),
|
||||
OperationEnum.EDIT,
|
||||
new Date(zkTopic.getCreateTime())
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// 这是删除的Topic
|
||||
for (Topic dbTopic: dbTopicMap.values()) {
|
||||
kafkaChangeRecordService.insertAndIgnoreDuplicate(new KafkaChangeRecordPO(
|
||||
clusterPhyId,
|
||||
ModuleEnum.KAFKA_TOPIC,
|
||||
dbTopic.getTopicName(),
|
||||
OperationEnum.DELETE,
|
||||
new Date(triggerTime),
|
||||
new Date(dbTopic.getCreateTime()) // 用被删除Topic的创建时间，避免重复记录删除动作
|
||||
));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=recordChangeToDB||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateDBData(Long clusterPhyId, List<Topic> topicList) {
|
||||
try {
|
||||
topicService.batchReplaceMetadata(clusterPhyId, topicList);
|
||||
} catch (Exception e) {
|
||||
log.error("method=updateDBData||clusterPhyId={}||errMsg=exception", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.acl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaAclPO;
|
||||
import org.apache.kafka.common.acl.AclBinding;
|
||||
import org.apache.kafka.common.resource.ResourceType;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface KafkaAclService {
|
||||
Result<List<AclBinding>> getAclFromKafka(Long clusterPhyId);
|
||||
|
||||
List<KafkaAclPO> getKafkaAclFromDB(Long clusterPhyId);
|
||||
|
||||
Integer countKafkaAclFromDB(Long clusterPhyId);
|
||||
|
||||
Integer countResTypeAndDistinctFromDB(Long clusterPhyId, ResourceType resourceType);
|
||||
|
||||
Integer countKafkaUserAndDistinctFromDB(Long clusterPhyId);
|
||||
|
||||
List<KafkaAclPO> getKafkaResTypeAclFromDB(Long clusterPhyId, Integer resType);
|
||||
|
||||
List<KafkaAclPO> getTopicAclFromDB(Long clusterPhyId, String topicName);
|
||||
|
||||
List<KafkaAclPO> getGroupAclFromDB(Long clusterPhyId, String groupName);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.acl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.acl.ACLAtomParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaAclPO;
|
||||
import org.apache.kafka.common.resource.ResourceType;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
public interface OpKafkaAclService {
|
||||
/**
|
||||
* 创建ACL
|
||||
*/
|
||||
Result<Void> createKafkaAcl(ACLAtomParam aclAtomParam, String operator);
|
||||
|
||||
/**
|
||||
* 删除ACL
|
||||
*/
|
||||
Result<Void> deleteKafkaAcl(ACLAtomParam aclAtomParam, String operator);
|
||||
|
||||
/**
|
||||
* 删除ACL
|
||||
*/
|
||||
Result<Void> deleteKafkaAclByResName(ResourceType resourceType, String resourceName, String operator);
|
||||
|
||||
Result<Void> insertAndIgnoreDuplicate(KafkaAclPO kafkaAclPO);
|
||||
|
||||
void batchUpdateAcls(Long clusterPhyId, List<KafkaAclPO> poList);
|
||||
|
||||
int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime);
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.acl.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaAclPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.acl.KafkaAclService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.KafkaAclDAO;
|
||||
import kafka.security.authorizer.AclAuthorizer;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import kafka.zk.ZkAclStore;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.DescribeAclsOptions;
|
||||
import org.apache.kafka.clients.admin.DescribeAclsResult;
|
||||
import org.apache.kafka.common.acl.*;
|
||||
import org.apache.kafka.common.resource.PatternType;
|
||||
import org.apache.kafka.common.resource.ResourcePattern;
|
||||
import org.apache.kafka.common.resource.ResourceType;
|
||||
import org.apache.kafka.common.security.auth.KafkaPrincipal;
|
||||
import org.apache.kafka.common.utils.SecurityUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import scala.jdk.javaapi.CollectionConverters;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
|
||||
|
||||
@Service
|
||||
public class KafkaAclServiceImpl extends BaseVersionControlService implements KafkaAclService {
|
||||
private static final ILog log = LogFactory.getLog(KafkaAclServiceImpl.class);
|
||||
|
||||
private static final String ACL_GET_FROM_KAFKA = "getAclFromKafka";
|
||||
|
||||
@Autowired
|
||||
private KafkaAclDAO kafkaAclDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.SERVICE_OP_ACL;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(ACL_GET_FROM_KAFKA, V_0_10_0_0, V_2_8_0, "getAclByZKClient", this::getAclByZKClient);
|
||||
registerVCHandler(ACL_GET_FROM_KAFKA, V_2_8_0, V_MAX, "getAclByKafkaClient", this::getAclByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<AclBinding>> getAclFromKafka(Long clusterPhyId) {
|
||||
if (LoadedClusterPhyCache.getByPhyId(clusterPhyId) == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterPhyId));
|
||||
}
|
||||
|
||||
try {
|
||||
return (Result<List<AclBinding>>) versionControlService.doHandler(getVersionItemType(), getMethodName(clusterPhyId, ACL_GET_FROM_KAFKA), new ClusterPhyParam(clusterPhyId));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KafkaAclPO> getKafkaAclFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<KafkaAclPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.orderByDesc(KafkaAclPO::getUpdateTime);
|
||||
return kafkaAclDAO.selectList(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countKafkaAclFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<KafkaAclPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
return kafkaAclDAO.selectCount(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countResTypeAndDistinctFromDB(Long clusterPhyId, ResourceType resourceType) {
|
||||
LambdaQueryWrapper<KafkaAclPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(KafkaAclPO::getResourceType, resourceType.code());
|
||||
lambdaQueryWrapper.ne(KafkaAclPO::getResourceName, "*"); // 等于*的不做统计
|
||||
|
||||
List<KafkaAclPO> poList = kafkaAclDAO.selectList(lambdaQueryWrapper);
|
||||
if (ValidateUtils.isEmptyList(poList)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return (int)poList.stream().map(elem -> elem.getResourceName()).distinct().count();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countKafkaUserAndDistinctFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<KafkaAclPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.ne(KafkaAclPO::getPrincipal, new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "*").toString()); // 等于*的不做统计
|
||||
|
||||
List<KafkaAclPO> poList = kafkaAclDAO.selectList(lambdaQueryWrapper);
|
||||
if (ValidateUtils.isEmptyList(poList)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return (int)poList.stream().map(elem -> elem.getPrincipal()).distinct().count();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KafkaAclPO> getKafkaResTypeAclFromDB(Long clusterPhyId, Integer resType) {
|
||||
LambdaQueryWrapper<KafkaAclPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.eq(KafkaAclPO::getResourceType, resType);
|
||||
return kafkaAclDAO.selectList(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KafkaAclPO> getTopicAclFromDB(Long clusterPhyId, String topicName) {
|
||||
// Topic自身 & Topic为*的情况
|
||||
LambdaQueryWrapper<KafkaAclPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.eq(KafkaAclPO::getResourceType, ResourceType.TOPIC.code());
|
||||
queryWrapper.and(qw -> qw.eq(KafkaAclPO::getResourceName, topicName).or().eq(KafkaAclPO::getResourceName, "*"));
|
||||
|
||||
return kafkaAclDAO.selectList(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KafkaAclPO> getGroupAclFromDB(Long clusterPhyId, String groupName) {
|
||||
LambdaQueryWrapper<KafkaAclPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.eq(KafkaAclPO::getResourceType, ResourceType.GROUP.code());
|
||||
queryWrapper.eq(KafkaAclPO::getResourceName, groupName);
|
||||
return kafkaAclDAO.selectList(queryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private Result<List<AclBinding>> getAclByZKClient(VersionItemParam itemParam){
|
||||
ClusterPhyParam param = (ClusterPhyParam) itemParam;
|
||||
|
||||
List<AclBinding> aclList = new ArrayList<>();
|
||||
for (ZkAclStore store: CollectionConverters.asJava(ZkAclStore.stores())) {
|
||||
Result<List<AclBinding>> rl = this.getSpecifiedTypeAclByZKClient(param.getClusterPhyId(), store.patternType());
|
||||
if (rl.failed()) {
|
||||
return rl;
|
||||
}
|
||||
|
||||
aclList.addAll(rl.getData());
|
||||
}
|
||||
|
||||
return Result.buildSuc(aclList);
|
||||
}
|
||||
|
||||
private Result<List<AclBinding>> getAclByKafkaClient(VersionItemParam itemParam) {
|
||||
ClusterPhyParam param = (ClusterPhyParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
DescribeAclsResult describeAclsResult =
|
||||
adminClient.describeAcls(AclBindingFilter.ANY, new DescribeAclsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
|
||||
return Result.buildSuc(new ArrayList<>(describeAclsResult.values().get()));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getAclByKafkaClient||clusterPhyId={}||errMsg={}", param.getClusterPhyId(), e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<AclBinding>> getSpecifiedTypeAclByZKClient(Long clusterPhyId, PatternType patternType) {
|
||||
List<AclBinding> kafkaAclList = new ArrayList<>();
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(clusterPhyId);
|
||||
|
||||
for (String rType: CollectionConverters.asJava(kafkaZkClient.getResourceTypes(patternType))) {
|
||||
ResourceType resourceType = SecurityUtils.resourceType(rType);
|
||||
for (String resourceName : CollectionConverters.asJava(kafkaZkClient.getResourceNames(patternType, resourceType))) {
|
||||
ResourcePattern resourcePattern = new ResourcePattern(resourceType, resourceName, patternType);
|
||||
AclAuthorizer.VersionedAcls versionedAcls = kafkaZkClient.getVersionedAclsForResource(resourcePattern);
|
||||
|
||||
CollectionConverters.asJava(versionedAcls.acls()).forEach(elem -> kafkaAclList.add(new AclBinding(resourcePattern, elem.ace())));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=getSpecifiedTypeAclByZKClient||clusterPhyId={}||patternType={}||errMsg={}", clusterPhyId, patternType, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc(kafkaAclList);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.acl.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.acl.ACLAtomParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaAclPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.KafkaAclConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.acl.OpKafkaAclService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.KafkaAclDAO;
|
||||
import kafka.security.authorizer.AclAuthorizer;
|
||||
import kafka.security.authorizer.AclEntry;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import kafka.zk.ZkVersion;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.acl.*;
|
||||
import org.apache.kafka.common.resource.ResourcePattern;
|
||||
import org.apache.kafka.common.resource.ResourcePatternFilter;
|
||||
import org.apache.kafka.common.resource.ResourceType;
|
||||
import org.apache.kafka.common.security.auth.KafkaPrincipal;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import scala.jdk.javaapi.CollectionConverters;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
|
||||
|
||||
@Service
|
||||
public class OpKafkaAclServiceImpl extends BaseVersionControlService implements OpKafkaAclService {
|
||||
private static final ILog log = LogFactory.getLog(OpKafkaAclServiceImpl.class);
|
||||
|
||||
private static final String ACL_CREATE = "createKafkaAcl";
|
||||
|
||||
private static final String ACL_DELETE = "deleteKafkaAcl";
|
||||
|
||||
@Autowired
|
||||
private KafkaAclDAO kafkaAclDAO;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.SERVICE_OP_ACL;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(ACL_CREATE, V_0_10_0_0, V_2_7_2, "createAclByZKClient", this::createAclByZKClient);
|
||||
registerVCHandler(ACL_CREATE, V_2_7_2, V_MAX, "createAclByKafkaClient", this::createAclByKafkaClient);
|
||||
|
||||
registerVCHandler(ACL_DELETE, V_0_10_0_0, V_2_7_2, "deleteAclByZKClient", this::deleteAclByZKClient);
|
||||
registerVCHandler(ACL_DELETE, V_2_7_2, V_MAX, "deleteAclByKafkaClient", this::deleteAclByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> createKafkaAcl(ACLAtomParam aclAtomParam, String operator) {
|
||||
if (ValidateUtils.anyNull(aclAtomParam, operator)) {
|
||||
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
|
||||
}
|
||||
|
||||
// 检查参数
|
||||
Result<Void> rv = aclAtomParam.checkFieldLegal();
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 创建ACL
|
||||
try {
|
||||
rv = (Result<Void>) versionControlService.doHandler(getVersionItemType(), getMethodName(aclAtomParam.getClusterPhyId(), ACL_CREATE), aclAtomParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
// 创建ACL失败
|
||||
return rv;
|
||||
}
|
||||
|
||||
KafkaAclPO po = KafkaAclConverter.convert2KafkaAclPO(aclAtomParam);
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.REPLACE.getDesc(),
|
||||
ModuleEnum.KAFKA_ACL.getDesc(),
|
||||
po.getUniqueField(),
|
||||
ConvertUtil.obj2Json(po)
|
||||
));
|
||||
|
||||
// 创建ACL成功，则将ACL信息写DB
|
||||
rv = this.insertAndIgnoreDuplicate(po);
|
||||
if (rv.failed()) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FAILED, "创建Kafka中的ACL信息成功，但是写入DB失败");
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> deleteKafkaAcl(ACLAtomParam aclAtomParam, String operator) {
|
||||
if (ValidateUtils.anyNull(aclAtomParam, operator)) {
|
||||
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
|
||||
}
|
||||
|
||||
// 检查参数
|
||||
Result<Void> rv = aclAtomParam.checkFieldLegal();
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 删除ACL
|
||||
try {
|
||||
rv = (Result<Void>) versionControlService.doHandler(getVersionItemType(), getMethodName(aclAtomParam.getClusterPhyId(), ACL_DELETE), aclAtomParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
// 删除ACL失败
|
||||
return rv;
|
||||
}
|
||||
|
||||
KafkaAclPO po = KafkaAclConverter.convert2KafkaAclPO(aclAtomParam);
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.DELETE.getDesc(),
|
||||
ModuleEnum.KAFKA_ACL.getDesc(),
|
||||
po.getUniqueField(),
|
||||
ConvertUtil.obj2Json(po)
|
||||
));
|
||||
|
||||
// 删除ACL成功，但是将ACL信息从DB中删除失败
|
||||
rv = this.deleteInDB(po);
|
||||
if (rv.failed()) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FAILED, "删除Kafka中的ACL成功，但是删除DB中的数据失败");
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> deleteKafkaAclByResName(ResourceType resourceType, String resourceName, String operator) {
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> insertAndIgnoreDuplicate(KafkaAclPO kafkaAclPO) {
|
||||
try {
|
||||
kafkaAclDAO.insert(kafkaAclPO);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (DuplicateKeyException dke) {
|
||||
// 直接写入，如果出现key冲突则直接忽略，因为key冲突时，表示该数据已完整存在，不需要替换任何数据
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=insertAndIgnoreDuplicate||kafkaAclPO={}||errMsg=exception", kafkaAclPO, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchUpdateAcls(Long clusterPhyId, List<KafkaAclPO> poList) {
|
||||
LambdaQueryWrapper<KafkaAclPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
Map<String, KafkaAclPO> dbPOMap = kafkaAclDAO.selectList(lambdaQueryWrapper).stream().collect(Collectors.toMap(KafkaAclPO::getUniqueField, Function.identity()));
|
||||
for (KafkaAclPO po: poList) {
|
||||
KafkaAclPO dbPO = dbPOMap.remove(po.getUniqueField());
|
||||
if (dbPO == null) {
|
||||
// 新增的ACL
|
||||
this.insertAndIgnoreDuplicate(po);
|
||||
}
|
||||
}
|
||||
|
||||
// 删除已经不存在的
|
||||
for (KafkaAclPO dbPO: dbPOMap.values()) {
|
||||
kafkaAclDAO.deleteById(dbPO);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime) {
|
||||
LambdaQueryWrapper<KafkaAclPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaAclPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.le(KafkaAclPO::getUpdateTime, beforeTime);
|
||||
return kafkaAclDAO.delete(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private Result<Void> deleteInDB(KafkaAclPO kafkaAclPO) {
|
||||
try {
|
||||
LambdaQueryWrapper<KafkaAclPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaAclPO::getUniqueField, kafkaAclPO.getUniqueField());
|
||||
kafkaAclDAO.delete(lambdaQueryWrapper);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=deleteInDB||kafkaAclPO={}||errMsg=exception", kafkaAclPO, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> createAclByZKClient(VersionItemParam itemParam) {
|
||||
return updateAclByZKClient((ACLAtomParam) itemParam, false);
|
||||
}
|
||||
|
||||
private Result<Void> createAclByKafkaClient(VersionItemParam itemParam) {
|
||||
ACLAtomParam param = (ACLAtomParam) itemParam;
|
||||
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
CreateAclsResult createAclsResult = adminClient.createAcls(
|
||||
Arrays.asList(new AclBinding(new ResourcePattern(param.getResourceType(), param.getResourceName(), param.getResourcePatternType()),
|
||||
new AccessControlEntry(
|
||||
new KafkaPrincipal(KafkaPrincipal.USER_TYPE, param.getKafkaUserName()).toString(),
|
||||
param.getAclClientHost(),
|
||||
param.getAclOperation(),
|
||||
param.getAclPermissionType()
|
||||
)
|
||||
)),
|
||||
new CreateAclsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
createAclsResult.all().get();
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=createAclByKafkaClient||parma={}||errMsg={}", itemParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> deleteAclByZKClient(VersionItemParam itemParam) {
|
||||
// 删除
|
||||
return this.updateAclByZKClient((ACLAtomParam) itemParam, true);
|
||||
}
|
||||
|
||||
private Result<Void> deleteAclByKafkaClient(VersionItemParam itemParam) {
|
||||
ACLAtomParam param = (ACLAtomParam) itemParam;
|
||||
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
// 构造需要删除的权限
|
||||
AclBindingFilter aclBindingFilter = new AclBindingFilter(
|
||||
new ResourcePatternFilter(param.getResourceType(), param.getResourceName(), param.getResourcePatternType()),
|
||||
new AccessControlEntryFilter(
|
||||
new KafkaPrincipal(KafkaPrincipal.USER_TYPE, param.getKafkaUserName()).toString(),
|
||||
param.getAclClientHost(),
|
||||
param.getAclOperation(),
|
||||
param.getAclPermissionType()
|
||||
)
|
||||
);
|
||||
|
||||
DeleteAclsResult deleteAclsResult = adminClient.deleteAcls(Arrays.asList(aclBindingFilter), new DeleteAclsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
deleteAclsResult.all().get();
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=deleteAclByKafkaClient||parma={}||errMsg={}", itemParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> updateAclByZKClient(ACLAtomParam aclAtomParam, boolean deleteAcl) {
|
||||
// 资源
|
||||
ResourcePattern resourcePattern = new ResourcePattern(
|
||||
aclAtomParam.getResourceType(),
|
||||
aclAtomParam.getResourceName(),
|
||||
aclAtomParam.getResourcePatternType()
|
||||
);
|
||||
|
||||
// 权限
|
||||
AclEntry aclEntry = new AclEntry(new AccessControlEntry(
|
||||
new KafkaPrincipal(KafkaPrincipal.USER_TYPE, aclAtomParam.getKafkaUserName()).toString(),
|
||||
aclAtomParam.getAclClientHost(),
|
||||
aclAtomParam.getAclOperation(),
|
||||
aclAtomParam.getAclPermissionType()
|
||||
));
|
||||
|
||||
// 当前AclAuthorizer中有现成的方法，但是AclAuthorizer会加载集群的ACL信息，会比较重
|
||||
// ZK客户端无现成可用方法，因此基于ZK客户端提供出来的方法，自定义实现操作ACL方法
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(aclAtomParam.getClusterPhyId());
|
||||
|
||||
// 创建ACL相关的ZK节点
|
||||
kafkaZkClient.createAclPaths();
|
||||
|
||||
Result<Boolean> rb = this.updateZKAcl(kafkaZkClient, resourcePattern, aclEntry, deleteAcl);
|
||||
if (rb.failed() || !rb.hasData()) {
|
||||
return Result.buildFromIgnoreData(rb);
|
||||
}
|
||||
|
||||
if (!rb.getData()) {
|
||||
// ACL操作失败
|
||||
return deleteAcl? Result.buildFailure("删除ACL失败"): Result.buildFailure("创建ACL失败");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=updateAclByZKClient||parma={}||deleteAcl={}||msg=update acl failed||errMsg={}", aclAtomParam, deleteAcl, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
// 操作成功的情况下，再尝试创建change节点
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(aclAtomParam.getClusterPhyId());
|
||||
|
||||
kafkaZkClient.createAclChangeNotification(resourcePattern);
|
||||
} catch (Exception e) {
|
||||
log.warn("method=updateAclByZKClient||parma={}||deleteAcl={}||msg=create change node failed||errMsg={}", aclAtomParam, deleteAcl, e.getMessage());
|
||||
|
||||
return deleteAcl? Result.buildFailure("删除ACL成功，但是创建Change节点失败"): Result.buildFailure("创建ACL成功，但是创建Change节点失败");
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Boolean> updateZKAcl(KafkaZkClient kafkaZkClient, ResourcePattern resourcePattern, AclEntry aclEntry, boolean deleteAcl) {
|
||||
// 获取当前资源的ACL信息
|
||||
AclAuthorizer.VersionedAcls versionedAcls = kafkaZkClient.getVersionedAclsForResource(resourcePattern);
|
||||
|
||||
Set<AclEntry> aclEntrySet = new HashSet<>(CollectionConverters.asJava(versionedAcls.acls()));
|
||||
|
||||
if ((deleteAcl && !aclEntrySet.contains(aclEntry))
|
||||
|| (!deleteAcl && aclEntrySet.contains(aclEntry))) {
|
||||
// 删除时，不存在该ACL则直接返回成功
|
||||
// 创建时，已存在该ACL则直接返回成功
|
||||
return Result.buildSuc(Boolean.TRUE);
|
||||
}
|
||||
|
||||
if (deleteAcl) {
|
||||
// 移除当前ACL
|
||||
aclEntrySet.remove(aclEntry);
|
||||
} else {
|
||||
// 增加当前ACL
|
||||
aclEntrySet.add(aclEntry);
|
||||
}
|
||||
|
||||
if (aclEntrySet.isEmpty()) {
|
||||
// 如果变更后为空，则删除节点
|
||||
return kafkaZkClient.deleteResource(resourcePattern)? Result.buildSuc(Boolean.TRUE): Result.buildSuc(Boolean.FALSE);
|
||||
} else if (ZkVersion.UnknownVersion() == versionedAcls.zkVersion()) {
|
||||
// 节点不存在
|
||||
return (boolean) kafkaZkClient.createAclsForResourceIfNotExists(
|
||||
resourcePattern,
|
||||
CollectionConverters.asScala(aclEntrySet).toSet()
|
||||
)._1()? Result.buildSuc(Boolean.TRUE): Result.buildSuc(Boolean.FALSE);
|
||||
} else {
|
||||
// 如果变更后非空，则更新节点
|
||||
return (boolean) kafkaZkClient.conditionalSetAclsForResource(
|
||||
resourcePattern,
|
||||
CollectionConverters.asScala(aclEntrySet).toSet(),
|
||||
versionedAcls.zkVersion()
|
||||
)._1() ? Result.buildSuc(Boolean.TRUE): Result.buildSuc(Boolean.FALSE);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaConfigDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaBrokerConfigModifyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerConfigPO;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Kafka相关配置接口
|
||||
* @author zengqiao
|
||||
* @date 22/03/03
|
||||
*/
|
||||
public interface BrokerConfigService {
|
||||
/**
|
||||
* 获取Broker配置
|
||||
*/
|
||||
Result<List<KafkaConfigDetail>> getBrokerConfigDetailFromKafka(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
Result<Void> modifyBrokerConfig(KafkaBrokerConfigModifyParam kafkaBrokerConfigModifyParam, String operator);
|
||||
|
||||
int countBrokerConfigDiffsFromDB(Long clusterPhyId, List<String> excludeConfigs);
|
||||
|
||||
List<BrokerConfigPO> getBrokerConfigDiffFromDB(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
int replaceBrokerConfigDiff(BrokerConfigPO po);
|
||||
|
||||
int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsBrokerDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public interface BrokerMetricService {
|
||||
|
||||
/**
|
||||
* 从Kafka获取指标
|
||||
*/
|
||||
Result<BrokerMetrics> collectBrokerMetricsFromKafkaWithCacheFirst(Long clusterPhyId, Integer brokerId, String metricName);
|
||||
Result<BrokerMetrics> collectBrokerMetricsFromKafka(Long clusterPhyId, Integer brokerId, String metricName);
|
||||
Result<BrokerMetrics> collectBrokerMetricsFromKafka(Long clusterPhyId, Integer brokerId, List<String> metricNames);
|
||||
|
||||
/**
|
||||
* 从ES中获取一段时间内聚合计算之后的指标线
|
||||
*/
|
||||
Result<List<MetricMultiLinesVO>> listBrokerMetricsFromES(Long clusterId, MetricsBrokerDTO dto);
|
||||
|
||||
/**
|
||||
* 从ES中获取聚合计算之后的一个指标点
|
||||
*/
|
||||
Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterPhyId, Integer brokerId, MetricDTO dto);
|
||||
|
||||
/**
|
||||
* 从ES中获取最近的一个指标
|
||||
*/
|
||||
Result<List<BrokerMetrics>> getLatestMetricsFromES(Long clusterPhyId, List<Integer> brokerIdList);
|
||||
|
||||
/**
|
||||
* 从ES中获取最近的一个指标
|
||||
*/
|
||||
Result<BrokerMetrics> getLatestMetricsFromES(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
boolean isMetricName(String str);
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import org.apache.kafka.clients.admin.LogDirDescription;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface BrokerService {
|
||||
/**
|
||||
* 从Kafka获取存活的Broker列表
|
||||
*/
|
||||
Result<List<Broker>> listBrokersFromKafka(ClusterPhy clusterPhy);
|
||||
|
||||
/**
|
||||
* 更新存活的Broker列表
|
||||
*/
|
||||
void updateAliveBrokers(Long clusterPhyId, List<Broker> presentAliveBrokerList);
|
||||
|
||||
/**
|
||||
* 从DB获取存活的Broker列表
|
||||
*/
|
||||
List<Broker> listAliveBrokersFromDB(Long clusterPhyId);
|
||||
List<Broker> listAliveBrokersFromCacheFirst(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 从DB获取未存活的Broker列表
|
||||
*/
|
||||
List<Broker> listNotAliveBrokersFromDB(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 从DB获取所有Broker列表
|
||||
*/
|
||||
List<Broker> listAllBrokersFromDB(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取Topic下的Broker列表
|
||||
*/
|
||||
List<Broker> listAllBrokerByTopic(Long clusterPhyId, String topicName);
|
||||
|
||||
/**
|
||||
* 获取具体Broker
|
||||
*/
|
||||
Broker getBroker(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
/**
|
||||
* 获取BrokerLog-Dir信息
|
||||
*/
|
||||
Result<Map<String, LogDirDescription>> getBrokerLogDirDescFromKafka(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
/**
|
||||
* 获取Broker的版本信息
|
||||
*/
|
||||
String getBrokerVersionFromKafka(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
/**
|
||||
* 获取总的Broker数
|
||||
*/
|
||||
Integer countAllBrokers();
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public interface BrokerSpecService {
|
||||
|
||||
/**
|
||||
*
|
||||
* @param clusterPhyId
|
||||
* @return
|
||||
*/
|
||||
Map<Integer, BrokerSpec> getBrokerSpecMap(Long clusterPhyId);
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaConfigDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.broker.BrokerParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaBrokerConfigModifyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerConfigPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.KafkaConfigConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerConfigService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerConfigDAO;
|
||||
import kafka.server.ConfigType;
|
||||
import kafka.zk.AdminZkClient;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.config.ConfigResource;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import scala.jdk.javaapi.CollectionConverters;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
|
||||
|
||||
@Service
|
||||
public class BrokerConfigServiceImpl extends BaseVersionControlService implements BrokerConfigService {
|
||||
private static final ILog log = LogFactory.getLog(BrokerConfigServiceImpl.class);
|
||||
|
||||
private static final String GET_BROKER_CONFIG = "getBrokerConfig";
|
||||
private static final String MODIFY_BROKER_CONFIG = "modifyBrokerConfig";
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Autowired
|
||||
private BrokerConfigDAO brokerConfigDAO;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.SERVICE_OP_BROKER_CONFIG;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(GET_BROKER_CONFIG, V_0_10_1_0, V_0_11_0_0, "getBrokerConfigByZKClient", this::getBrokerConfigByZKClient);
|
||||
registerVCHandler(GET_BROKER_CONFIG, V_0_11_0_0, V_MAX, "getBrokerConfigByKafkaClient", this::getBrokerConfigByKafkaClient);
|
||||
|
||||
registerVCHandler(MODIFY_BROKER_CONFIG, V_0_10_1_0, V_0_11_0_0, "modifyBrokerConfigByZKClient", this::modifyBrokerConfigByZKClient);
|
||||
registerVCHandler(MODIFY_BROKER_CONFIG, V_0_11_0_0, V_MAX, "modifyBrokerConfigByKafkaClient", this::modifyBrokerConfigByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<KafkaConfigDetail>> getBrokerConfigDetailFromKafka(Long clusterPhyId, Integer brokerId) {
|
||||
try {
|
||||
return (Result<List<KafkaConfigDetail>>) doVCHandler(clusterPhyId, GET_BROKER_CONFIG, new BrokerParam(clusterPhyId, brokerId));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> modifyBrokerConfig(KafkaBrokerConfigModifyParam kafkaBrokerConfigModifyParam, String operator) {
|
||||
Result<Void> rv = null;
|
||||
try {
|
||||
rv = (Result<Void>) versionControlService.doHandler(getVersionItemType(),
|
||||
getMethodName(kafkaBrokerConfigModifyParam.getClusterPhyId(), MODIFY_BROKER_CONFIG), kafkaBrokerConfigModifyParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.KAFKA_BROKER_CONFIG.getDesc(),
|
||||
MsgConstant.getBrokerBizStr(kafkaBrokerConfigModifyParam.getClusterPhyId(), kafkaBrokerConfigModifyParam.getBrokerId()),
|
||||
ConvertUtil.obj2Json(kafkaBrokerConfigModifyParam)
|
||||
));
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int countBrokerConfigDiffsFromDB(Long clusterPhyId, List<String> excludeConfigs) {
|
||||
LambdaQueryWrapper<BrokerConfigPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(BrokerConfigPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.notIn(BrokerConfigPO::getConfigName, excludeConfigs);
|
||||
|
||||
return brokerConfigDAO.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<BrokerConfigPO> getBrokerConfigDiffFromDB(Long clusterPhyId, Integer brokerId) {
|
||||
LambdaQueryWrapper<BrokerConfigPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(BrokerConfigPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(BrokerConfigPO::getBrokerId, brokerId);
|
||||
|
||||
return brokerConfigDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int replaceBrokerConfigDiff(BrokerConfigPO po) {
|
||||
return brokerConfigDAO.replace(po);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime) {
|
||||
LambdaQueryWrapper<BrokerConfigPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(BrokerConfigPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.le(BrokerConfigPO::getUpdateTime, beforeTime);
|
||||
|
||||
return brokerConfigDAO.delete(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private DescribeConfigsOptions buildDescribeConfigsOptions() {
|
||||
return new DescribeConfigsOptions().includeDocumentation(false).includeSynonyms(false).timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS);
|
||||
}
|
||||
|
||||
private Result<List<KafkaConfigDetail>> getBrokerConfigByZKClient(VersionItemParam itemParam) {
|
||||
BrokerParam param = (BrokerParam) itemParam;
|
||||
Result<Properties> propertiesResult = this.getBrokerConfigByZKClient(param.getClusterPhyId(), param.getBrokerId());
|
||||
if (propertiesResult.failed()) {
|
||||
return Result.buildFromIgnoreData(propertiesResult);
|
||||
}
|
||||
|
||||
return Result.buildSuc(KafkaConfigConverter.convert2KafkaBrokerConfigDetailList(
|
||||
new ArrayList<>(),
|
||||
propertiesResult.getData()
|
||||
));
|
||||
}
|
||||
|
||||
private Result<Properties> getBrokerConfigByZKClient(Long clusterPhyId, Integer brokerId) {
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(clusterPhyId);
|
||||
|
||||
Properties properties = kafkaZkClient.getEntityConfigs(ConfigType.Broker(), String.valueOf(brokerId));
|
||||
for (Object key: properties.keySet()) {
|
||||
properties.getProperty((String) key);
|
||||
}
|
||||
|
||||
return Result.buildSuc(properties);
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getBrokerNotExist(clusterPhyId, brokerId));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getBrokerConfigByZKClient||clusterPhyId={}||brokerId={}||errMsg=exception", clusterPhyId, brokerId, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<KafkaConfigDetail>> getBrokerConfigByKafkaClient(VersionItemParam itemParam) {
|
||||
BrokerParam param = (BrokerParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
ConfigResource configResource = new ConfigResource(ConfigResource.Type.BROKER, String.valueOf(param.getBrokerId()));
|
||||
DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(
|
||||
Arrays.asList(configResource),
|
||||
buildDescribeConfigsOptions()
|
||||
);
|
||||
|
||||
Map<ConfigResource, Config> configMap = describeConfigsResult.all().get();
|
||||
|
||||
return Result.buildSuc(KafkaConfigConverter.convert2KafkaConfigDetailList(
|
||||
configMap.get(configResource)
|
||||
));
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getBrokerNotExist(param.getClusterPhyId(), param.getBrokerId()));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getBrokerConfigByKafkaClient||param={}||errMsg=exception!", param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> modifyBrokerConfigByZKClient(VersionItemParam itemParam) {
|
||||
KafkaBrokerConfigModifyParam configParam = (KafkaBrokerConfigModifyParam) itemParam;
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(configParam.getClusterPhyId());
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(configParam.getClusterPhyId());
|
||||
|
||||
// 获取ZK配置
|
||||
Properties properties = kafkaZkClient.getEntityConfigs(ConfigType.Broker(), String.valueOf(configParam.getBrokerId()));
|
||||
properties.putAll(configParam.getChangedProps());
|
||||
|
||||
// 修改配置
|
||||
adminZkClient.changeBrokerConfig(CollectionConverters.asScala(Arrays.asList(configParam.getBrokerId())), properties);
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyBrokerConfigByZKClientAndNodeVersionV2||param={}||errMsg=exception.", configParam, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> modifyBrokerConfigByKafkaClient(VersionItemParam itemParam) {
|
||||
KafkaBrokerConfigModifyParam configParam = (KafkaBrokerConfigModifyParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(configParam.getClusterPhyId());
|
||||
|
||||
List<AlterConfigOp> configOpList = new ArrayList<>();
|
||||
for (Map.Entry<String, String> changedConfig: configParam.getChangedProps().entrySet()) {
|
||||
configOpList.add(new AlterConfigOp(new ConfigEntry(changedConfig.getKey(), changedConfig.getValue()), AlterConfigOp.OpType.SET));
|
||||
}
|
||||
|
||||
Map<ConfigResource, Collection<AlterConfigOp>> configMap = new HashMap<>();
|
||||
configMap.put(new ConfigResource(ConfigResource.Type.BROKER, String.valueOf(configParam.getBrokerId())), configOpList);
|
||||
|
||||
AlterConfigsResult alterConfigsResult = adminClient.incrementalAlterConfigs(configMap, new AlterConfigsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
alterConfigsResult.all().get();
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyBrokerConfigByKafkaClient||param={}||errMsg=exception.", configParam, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
return Result.buildSuc();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.google.common.collect.HashBasedTable;
|
||||
import com.google.common.collect.Table;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsBrokerDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.BrokerMetricParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.BrokerMetricPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BeanUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.cache.CollectMetricsLocalCache;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.score.HealthScoreService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.replica.ReplicaMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.BrokerMetricVersionItems;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.ReplicaMetricVersionItems;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.BrokerMetricESDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.management.InstanceNotFoundException;
|
||||
import javax.management.ObjectName;
|
||||
import javax.management.Query;
|
||||
import javax.management.QueryExp;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service
|
||||
public class BrokerMetricServiceImpl extends BaseMetricService implements BrokerMetricService {
|
||||
protected static final ILog LOGGER = LogFactory.getLog("METRIC_LOGGER");
|
||||
|
||||
public static final String BROKER_METHOD_DO_NOTHING = "doNothing";
|
||||
public static final String BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX = "getMetricFromKafkaByJMX";
|
||||
public static final String BROKER_METHOD_GET_CONNECTION_FROM_KAFKA_BY_JMX = "getConnectCountFromKafkaByJMX";
|
||||
public static final String BROKER_METHOD_GET_HEALTH_SCORE = "getMetricHealthScore";
|
||||
public static final String BROKER_METHOD_GET_PARTITIONS_SKEW = "getPartitionsSkew";
|
||||
public static final String BROKER_METHOD_GET_LEADERS_SKEW = "getLeadersSkew";
|
||||
public static final String BROKER_METHOD_GET_LOG_SIZE = "getLogSize";
|
||||
public static final String BROKER_METHOD_IS_BROKER_ALIVE = "isBrokerAlive";
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private ReplicaMetricService replicaMetricService;
|
||||
|
||||
@Autowired
|
||||
private HealthScoreService healthScoreService;
|
||||
|
||||
@Autowired
|
||||
private KafkaJMXClient kafkaJMXClient;
|
||||
|
||||
@Autowired
|
||||
private BrokerMetricESDAO brokerMetricESDAO;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.METRIC_BROKER;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<String> listMetricPOFields(){
|
||||
return BeanUtil.listBeanFields(BrokerMetricPO.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initRegisterVCHandler(){
|
||||
registerVCHandler( BROKER_METHOD_DO_NOTHING, this::doNothing);
|
||||
registerVCHandler( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX, this::getMetricFromKafkaByJMX);
|
||||
registerVCHandler( BROKER_METHOD_GET_CONNECTION_FROM_KAFKA_BY_JMX, this::getConnectCountFromKafkaByJMX);
|
||||
registerVCHandler( BROKER_METHOD_GET_HEALTH_SCORE, this::getMetricHealthScore);
|
||||
registerVCHandler( BROKER_METHOD_GET_PARTITIONS_SKEW, this::getPartitionsSkew);
|
||||
registerVCHandler( BROKER_METHOD_GET_LEADERS_SKEW, this::getLeadersSkew);
|
||||
registerVCHandler( BROKER_METHOD_GET_LOG_SIZE, this::getLogSize);
|
||||
registerVCHandler( BROKER_METHOD_IS_BROKER_ALIVE, this::isBrokerAlive);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<BrokerMetrics> collectBrokerMetricsFromKafkaWithCacheFirst(Long clusterId, Integer brokerId, String metric){
|
||||
|
||||
Float keyValue = CollectMetricsLocalCache.getBrokerMetrics(clusterId, brokerId, metric);
|
||||
if(null != keyValue) {
|
||||
BrokerMetrics brokerMetrics = new BrokerMetrics(clusterId, brokerId);
|
||||
brokerMetrics.putMetric(metric, keyValue);
|
||||
return Result.buildSuc(brokerMetrics);
|
||||
}
|
||||
|
||||
Result<BrokerMetrics> ret = this.collectBrokerMetricsFromKafka(clusterId, brokerId, metric);
|
||||
if(null == ret || ret.failed() || null == ret.getData()) {return ret;}
|
||||
|
||||
Map<String, Float> metricsMap = ret.getData().getMetrics();
|
||||
for(Map.Entry<String, Float> metricNameAndValueEntry : metricsMap.entrySet()){
|
||||
CollectMetricsLocalCache.putBrokerMetrics(clusterId, brokerId, metricNameAndValueEntry.getKey(), metricNameAndValueEntry.getValue());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<BrokerMetrics> collectBrokerMetricsFromKafka(Long clusterId, Integer brokerId, String metric){
|
||||
try {
|
||||
BrokerMetricParam brokerMetricParam = new BrokerMetricParam(clusterId, brokerId, metric );
|
||||
return (Result<BrokerMetrics>) doVCHandler(clusterId, metric, brokerMetricParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<BrokerMetrics> collectBrokerMetricsFromKafka(Long clusterId, Integer brokerId, List<String> metrics) {
|
||||
BrokerMetrics brokerMetrics = new BrokerMetrics(clusterId, brokerId);
|
||||
|
||||
for(String metric : metrics){
|
||||
if(null != brokerMetrics.getMetrics().get(metric)){continue;}
|
||||
|
||||
Result<BrokerMetrics> ret = collectBrokerMetricsFromKafka(clusterId, brokerId, metric);
|
||||
if(null != ret && !ret.failed()){
|
||||
brokerMetrics.putMetric(ret.getData().getMetrics());
|
||||
}
|
||||
}
|
||||
return Result.buildSuc(brokerMetrics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricMultiLinesVO>> listBrokerMetricsFromES(Long clusterId, MetricsBrokerDTO dto){
|
||||
Long startTime = dto.getStartTime();
|
||||
Long endTime = dto.getEndTime();
|
||||
Integer topN = dto.getTopNu();
|
||||
String aggType = dto.getAggType();
|
||||
List<Long> brokerIds = dto.getBrokerIds();
|
||||
List<String> metrics = dto.getMetricsNames();
|
||||
|
||||
Table<String/*metric*/, Long/*brokerId*/, List<MetricPointVO>> retTable;
|
||||
if(CollectionUtils.isEmpty(brokerIds)){
|
||||
List<Long> defaultBrokerIds = listTopNBrokerIds(clusterId, topN);
|
||||
retTable = brokerMetricESDAO.listBrokerMetricsByTop(clusterId, defaultBrokerIds, metrics, aggType, topN, startTime, endTime);
|
||||
}else {
|
||||
retTable = brokerMetricESDAO.listBrokerMetricsByBrokerIds(clusterId, metrics, aggType, brokerIds, startTime, endTime);
|
||||
}
|
||||
|
||||
List<MetricMultiLinesVO> multiLinesVOS = metricMap2VO(clusterId, convertId2Host(clusterId, retTable).rowMap());
|
||||
return Result.buildSuc(multiLinesVOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterPhyId, Integer brokerId, MetricDTO dto) {
|
||||
Map<String/*metric*/, MetricPointVO> metricPointMap = brokerMetricESDAO.getBrokerMetricsPoint(clusterPhyId, brokerId,
|
||||
dto.getMetricsNames(), dto.getAggType(), dto.getStartTime(), dto.getEndTime());
|
||||
|
||||
List<MetricPointVO> metricPoints = new ArrayList<>(metricPointMap.values());
|
||||
return Result.buildSuc(metricPoints);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<BrokerMetrics>> getLatestMetricsFromES(Long clusterPhyId, List<Integer> brokerIdList) {
|
||||
List<BrokerMetrics> brokerMetrics = new ArrayList<>();
|
||||
|
||||
for(Integer brokerId : brokerIdList) {
|
||||
try {
|
||||
BrokerMetricPO brokerMetricPO = brokerMetricESDAO.getBrokerLatestMetrics(clusterPhyId, brokerId);
|
||||
if (brokerMetricPO == null) {
|
||||
// 如果为null，则忽略该po
|
||||
continue;
|
||||
}
|
||||
|
||||
brokerMetrics.add(ConvertUtil.obj2Obj(brokerMetricPO, BrokerMetrics.class));
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=getLatestMetricsFromES||clusterPhyId={}||brokerId={}||errMsg=exception",
|
||||
clusterPhyId, brokerId, e);
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(brokerMetrics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<BrokerMetrics> getLatestMetricsFromES(Long clusterPhyId, Integer brokerId) {
|
||||
BrokerMetricPO brokerMetricPO = brokerMetricESDAO.getBrokerLatestMetrics(clusterPhyId, brokerId);
|
||||
return Result.buildSuc(ConvertUtil.obj2Obj(brokerMetricPO, BrokerMetrics.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMetricName(String str) {
|
||||
return super.isMetricName(str);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
private List<Long> listTopNBrokerIds(Long clusterId, Integer topN){
|
||||
List<Broker> brokers = brokerService.listAliveBrokersFromDB(clusterId);
|
||||
if(CollectionUtils.isEmpty(brokers)){return new ArrayList<>();}
|
||||
|
||||
return brokers.subList(0, Math.min(topN, brokers.size()))
|
||||
.stream().map(b -> b.getBrokerId().longValue()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> getConnectCountFromKafkaByJMX(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
//2、获取jmx连接
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, brokerId);
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){return Result.buildFailure(VC_JMX_INIT_ERROR);}
|
||||
|
||||
try {
|
||||
ObjectName queryListenerRegX = new ObjectName(jmxInfo.getJmxObjectName() + ",listener=*,networkProcessor=*");
|
||||
QueryExp exp = Query.gt(Query.attr(jmxInfo.getJmxAttribute()), Query.value(0.0));
|
||||
|
||||
Set<ObjectName> objectNames = jmxConnectorWrap.queryNames(queryListenerRegX, exp);
|
||||
float totalConnections = 0f;
|
||||
for(ObjectName objectName : objectNames){
|
||||
totalConnections += Double.valueOf(jmxConnectorWrap.getAttribute(objectName, jmxInfo.getJmxAttribute()).toString());
|
||||
}
|
||||
|
||||
BrokerMetrics brokerMetrics = new BrokerMetrics(clusterId, brokerId);
|
||||
brokerMetrics.putMetric(metric, totalConnections);
|
||||
return Result.buildSuc(brokerMetrics);
|
||||
} catch (InstanceNotFoundException e) {
|
||||
return Result.buildFailure(VC_JMX_INSTANCE_NOT_FOUND);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=getConnectCountFromKafkaByJMX||cluster={}||brokerId={}||metrics={}||jmx={}" +
|
||||
"||msg={}", clusterId, brokerId, metric, jmxInfo.getJmxObjectName(), e.getClass().getName());
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> getMetricFromKafkaByJMX(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
//2、获取jmx连接
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, brokerId);
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){return Result.buildFailure(VC_JMX_INIT_ERROR);}
|
||||
|
||||
try {
|
||||
//2、获取jmx指标
|
||||
String value = jmxConnectorWrap.getAttribute(new ObjectName(jmxInfo.getJmxObjectName()), jmxInfo.getJmxAttribute()).toString();
|
||||
|
||||
BrokerMetrics brokerMetrics = new BrokerMetrics(clusterId, brokerId);
|
||||
brokerMetrics.putMetric(metric, Float.valueOf(value));
|
||||
|
||||
return Result.buildSuc(brokerMetrics);
|
||||
} catch (Exception e) {
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 broker 的健康分
|
||||
* @param metricParam
|
||||
* @return
|
||||
*/
|
||||
private Result<BrokerMetrics> getMetricHealthScore(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
BrokerMetrics brokerMetrics = healthScoreService.calBrokerHealthScore(clusterId, brokerId);
|
||||
return Result.buildSuc(brokerMetrics);
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> getPartitionsSkew(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
Result<BrokerMetrics> metricsResult = this.collectBrokerMetricsFromKafkaWithCacheFirst(clusterId, brokerId, BrokerMetricVersionItems.BROKER_METRIC_PARTITIONS);
|
||||
if (metricsResult.failed()) {
|
||||
return metricsResult;
|
||||
}
|
||||
|
||||
Float brokerReplicaCount = metricsResult.getData().getMetric( BrokerMetricVersionItems.BROKER_METRIC_PARTITIONS);
|
||||
|
||||
Integer globalReplicaCount = topicService.getReplicaSizeFromCacheFirst(clusterId);
|
||||
|
||||
Integer globalBrokerCount = brokerService.listAllBrokersFromDB(clusterId).size();
|
||||
if (globalReplicaCount <= 0 || globalBrokerCount <= 0) {
|
||||
// 集群无分区或者集群无broker，则倾斜率直接设置为0
|
||||
return Result.buildSuc(BrokerMetrics.initWithMetric(
|
||||
clusterId,
|
||||
brokerId,
|
||||
metric,
|
||||
0f)
|
||||
);
|
||||
}
|
||||
|
||||
Float agvReplicaCount = globalReplicaCount.floatValue() / globalBrokerCount;
|
||||
return Result.buildSuc(BrokerMetrics.initWithMetric(
|
||||
clusterId,
|
||||
brokerId,
|
||||
metric,
|
||||
(brokerReplicaCount - agvReplicaCount) / agvReplicaCount)
|
||||
);
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> getLogSize(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
List<Partition> partitions = partitionService.listPartitionByBroker(clusterId, brokerId);
|
||||
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, brokerId);
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){return Result.buildFailure(VC_JMX_INIT_ERROR);}
|
||||
|
||||
Float logSizeSum = 0f;
|
||||
for(Partition p : partitions) {
|
||||
try {
|
||||
Result<ReplicationMetrics> metricsResult = replicaMetricService.collectReplicaMetricsFromKafkaWithCache(
|
||||
clusterId,
|
||||
p.getTopicName(),
|
||||
brokerId,
|
||||
p.getPartitionId(),
|
||||
ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE
|
||||
);
|
||||
if(null == metricsResult || metricsResult.failed() || null == metricsResult.getData()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Float replicaLogSize = metricsResult.getData().getMetric(ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE);
|
||||
|
||||
logSizeSum += (replicaLogSize == null? 0.0f: replicaLogSize);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(
|
||||
"class=BrokerMetricServiceImpl||method=getLogSize||clusterPhyId={}||brokerId={}||topicName={}||partitionId={}||metricName={}||errMsg=exception",
|
||||
clusterId, brokerId, p.getTopicName(), p.getPartitionId(), metric, e.getClass().getName()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(BrokerMetrics.initWithMetric(clusterId, brokerId, metric, logSizeSum));
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> getLeadersSkew(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
Result<BrokerMetrics> metricsResult = this.collectBrokerMetricsFromKafkaWithCacheFirst(clusterId, brokerId, BrokerMetricVersionItems.BROKER_METRIC_LEADERS);
|
||||
if (metricsResult.failed()) {
|
||||
return metricsResult;
|
||||
}
|
||||
|
||||
Float brokerLeaderCount = metricsResult.getData().getMetric( BrokerMetricVersionItems.BROKER_METRIC_LEADERS);
|
||||
|
||||
Integer globalLeaderCount = partitionService.getLeaderPartitionSizeByClusterId(clusterId);
|
||||
|
||||
Integer globalBrokerCount = brokerService.listAllBrokersFromDB(clusterId).size();
|
||||
if (globalLeaderCount <= 0 || globalBrokerCount <= 0) {
|
||||
// 集群无leader分区或者集群无broker，则倾斜率直接设置为0
|
||||
return Result.buildSuc(BrokerMetrics.initWithMetric(
|
||||
clusterId,
|
||||
brokerId,
|
||||
metric,
|
||||
0f)
|
||||
);
|
||||
}
|
||||
|
||||
Float agvPartitionCount = globalLeaderCount.floatValue() / globalBrokerCount;
|
||||
return Result.buildSuc(BrokerMetrics.initWithMetric(
|
||||
clusterId,
|
||||
brokerId,
|
||||
metric,
|
||||
(brokerLeaderCount - agvPartitionCount) / agvPartitionCount)
|
||||
);
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> isBrokerAlive(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer brokerId = param.getBrokerId();
|
||||
|
||||
Broker broker = brokerService.getBroker(clusterId, brokerId);
|
||||
if (broker == null) {
|
||||
return Result.buildFromRSAndMsg(NOT_EXIST, MsgConstant.getBrokerNotExist(clusterId, brokerId));
|
||||
}
|
||||
|
||||
return Result.buildSuc(BrokerMetrics.initWithMetric(clusterId, brokerId, metric, broker.alive()? Constant.YES.floatValue(): Constant.NO.floatValue()));
|
||||
}
|
||||
|
||||
private Result<BrokerMetrics> doNothing(VersionItemParam metricParam) {
|
||||
BrokerMetricParam param = (BrokerMetricParam)metricParam;
|
||||
return Result.buildSuc(new BrokerMetrics(param.getClusterId(), param.getBrokerId()));
|
||||
}
|
||||
|
||||
private Table<String/*metric*/, String/*host:port*/, List<MetricPointVO>> convertId2Host(
|
||||
Long clusterId,
|
||||
Table<String/*metric*/, Long/*brokerId*/, List<MetricPointVO>> table){
|
||||
Table<String, String, List<MetricPointVO>> destTable = HashBasedTable.create();
|
||||
|
||||
Map<Long, Broker> brokerMap = new HashMap<>();
|
||||
|
||||
for(String metric : table.rowKeySet()){
|
||||
Map<Long, List<MetricPointVO>> brokerIdMetricPointMap = table.row(metric);
|
||||
for(Long brokerId : brokerIdMetricPointMap.keySet()){
|
||||
Broker broker = brokerMap.getOrDefault(brokerId, brokerService.getBroker(clusterId, brokerId.intValue()));
|
||||
brokerMap.put(brokerId, broker);
|
||||
destTable.put(metric, broker.getHost() + ":" + broker.getPort(), brokerIdMetricPointMap.get(brokerId));
|
||||
}
|
||||
}
|
||||
|
||||
return destTable;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.broker.BrokerParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.broker.BrokerPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterRunStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.BrokerMetadata;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.broker.BrokerDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.BrokerIdZNode;
|
||||
import kafka.zk.BrokerIdsZNode;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.management.ObjectName;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.VC_HANDLE_NOT_EXIST;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.V_1_0_0;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.V_MAX;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.SERVICE_SEARCH_BROKER;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.VERSION;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxName.JMX_SERVER_APP_INFO;
|
||||
|
||||
@Service
|
||||
public class BrokerServiceImpl extends BaseVersionControlService implements BrokerService {
|
||||
private static final ILog log = LogFactory.getLog(BrokerServiceImpl.class);
|
||||
|
||||
private static final String BROKER_LOG_DIR = "getLogDir";
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaJMXClient kafkaJMXClient;
|
||||
|
||||
@Autowired
|
||||
private JmxDAO jmxDAO;
|
||||
|
||||
@Autowired
|
||||
private BrokerDAO brokerDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return SERVICE_SEARCH_BROKER;
|
||||
}
|
||||
|
||||
private final Cache<Long, List<Broker>> brokersCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(90, TimeUnit.SECONDS)
|
||||
.maximumSize(200)
|
||||
.build();
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(BROKER_LOG_DIR, V_1_0_0, V_MAX, "getLogDirByKafkaClient", this::getLogDirByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<Broker>> listBrokersFromKafka(ClusterPhy clusterPhy) {
|
||||
if (clusterPhy.getRunState().equals(ClusterRunStateEnum.RUN_ZK.getRunState())) {
|
||||
return this.getBrokersFromZKClient(clusterPhy);
|
||||
}
|
||||
|
||||
return this.getBrokersFromAdminClient(clusterPhy);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateAliveBrokers(Long clusterPhyId, List<Broker> presentAliveBrokerList) {
|
||||
long now = System.currentTimeMillis();
|
||||
|
||||
Map<Integer, Broker> presentAliveMap = presentAliveBrokerList.stream().collect(Collectors.toMap(Broker::getBrokerId, Function.identity()));
|
||||
|
||||
List<BrokerPO> inDBBrokerPOList = this.getAllBrokerPOsFromDB(clusterPhyId);
|
||||
for (BrokerPO inDBBrokerPO: inDBBrokerPOList) {
|
||||
Broker presentAliveBroker = presentAliveMap.remove(inDBBrokerPO.getBrokerId());
|
||||
if (presentAliveBroker == null && Constant.DOWN.equals(inDBBrokerPO.getStatus())) {
|
||||
continue;
|
||||
} else if (presentAliveBroker == null && Constant.ALIVE.equals(inDBBrokerPO.getStatus())) {
|
||||
// 当前Broker已经挂了，但是DB中显示其存活，则将其状态设置为0
|
||||
inDBBrokerPO.setStatus(Constant.DOWN);
|
||||
inDBBrokerPO.setStartTimestamp(now); // 设置挂掉时间
|
||||
brokerDAO.updateById(inDBBrokerPO);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 如果当前Broker还存活，则更新DB信息
|
||||
BrokerPO newBrokerPO = ConvertUtil.obj2Obj(presentAliveBroker, BrokerPO.class);
|
||||
newBrokerPO.setId(inDBBrokerPO.getId());
|
||||
newBrokerPO.setStatus(Constant.ALIVE);
|
||||
newBrokerPO.setCreateTime(inDBBrokerPO.getCreateTime());
|
||||
newBrokerPO.setUpdateTime(inDBBrokerPO.getUpdateTime());
|
||||
if (newBrokerPO.getStartTimestamp() == null) {
|
||||
// 如果当前broker获取不到启动时间
|
||||
// 如果DB中的broker状态为down，则使用当前时间，否则使用db中已有broker的时间
|
||||
newBrokerPO.setStartTimestamp(inDBBrokerPO.getStatus().equals(0)? now: inDBBrokerPO.getStartTimestamp());
|
||||
}
|
||||
brokerDAO.updateById(newBrokerPO);
|
||||
}
|
||||
|
||||
// 将presentAliveMap中剩下的Broker插入到DB中
|
||||
for (Broker presentAliveBroker: presentAliveMap.values()) {
|
||||
try {
|
||||
if (presentAliveBroker.getStartTimestamp() == null) {
|
||||
presentAliveBroker.setStartTimestamp(now);
|
||||
}
|
||||
brokerDAO.insert(ConvertUtil.obj2Obj(presentAliveBroker, BrokerPO.class));
|
||||
} catch (DuplicateKeyException dke) {
|
||||
// 因为有多台KM一起执行，因此可能存在key冲突的问题，如果出现则直接忽略该错误
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Broker> listAliveBrokersFromDB(Long clusterPhyId) {
|
||||
return this.listAllBrokersAndUpdateCache(clusterPhyId).stream().filter( elem -> elem.alive()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Broker> listAliveBrokersFromCacheFirst(Long clusterPhyId) {
|
||||
List<Broker> allBrokerList = brokersCache.getIfPresent(clusterPhyId);
|
||||
if (allBrokerList == null) {
|
||||
allBrokerList = this.listAllBrokersAndUpdateCache(clusterPhyId);
|
||||
}
|
||||
|
||||
return allBrokerList.stream().filter( elem -> elem.alive()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Broker> listNotAliveBrokersFromDB(Long clusterPhyId) {
|
||||
return this.listAllBrokersAndUpdateCache(clusterPhyId).stream().filter( elem -> !elem.alive()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Broker> listAllBrokersFromDB(Long clusterPhyId) {
|
||||
return this.listAllBrokersAndUpdateCache(clusterPhyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Broker> listAllBrokerByTopic(Long clusterPhyId, String topicName) {
|
||||
List<Broker> brokerList = this.listAllBrokersFromDB(clusterPhyId);
|
||||
try {
|
||||
Topic topic = topicService.getTopic(clusterPhyId, topicName);
|
||||
if (topic == null) {
|
||||
return brokerList;
|
||||
}
|
||||
|
||||
return brokerList.stream().filter(elem -> topic.getBrokerIdSet().contains(elem.getBrokerId())).collect(Collectors.toList());
|
||||
} catch (Exception e) {
|
||||
log.error("method=listAllBrokerByTopic||clusterPhyId={}||topicName={}||errMsg=exception!", clusterPhyId, topicName, e);
|
||||
}
|
||||
|
||||
return brokerList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Broker getBroker(Long clusterPhyId, Integer brokerId) {
|
||||
LambdaQueryWrapper<BrokerPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(BrokerPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(BrokerPO::getBrokerId, brokerId);
|
||||
|
||||
return ConvertUtil.obj2Obj(brokerDAO.selectOne(lambdaQueryWrapper), Broker.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<String, LogDirDescription>> getBrokerLogDirDescFromKafka(Long clusterPhyId, Integer brokerId) {
|
||||
try {
|
||||
return (Result<Map<String, LogDirDescription>>) doVCHandler(clusterPhyId, BROKER_LOG_DIR, new BrokerParam(clusterPhyId, brokerId));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getBrokerVersionFromKafka(Long clusterId, Integer brokerId) {
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClient(clusterId, brokerId);
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap) || !jmxConnectorWrap.checkJmxConnectionAndInitIfNeed()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
return (String) jmxConnectorWrap.getAttribute(new ObjectName(JMX_SERVER_APP_INFO + ",id=" + brokerId), VERSION);
|
||||
} catch (Exception e) {
|
||||
log.error("method=collectBrokerVersionFromKafka||clusterId:{}||brokerId:{}||errMsg=exception.", clusterId, brokerId, e);
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countAllBrokers() {
|
||||
LambdaQueryWrapper<BrokerPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
return brokerDAO.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private List<Broker> listAllBrokersAndUpdateCache(Long clusterPhyId) {
|
||||
List<Broker> allBrokerList = ConvertUtil.list2List(this.getAllBrokerPOsFromDB(clusterPhyId), Broker.class);
|
||||
brokersCache.put(clusterPhyId, allBrokerList);
|
||||
|
||||
return allBrokerList;
|
||||
}
|
||||
|
||||
private List<BrokerPO> getAllBrokerPOsFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<BrokerPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(BrokerPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return brokerDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
private Result<Map<String, LogDirDescription>> getLogDirByKafkaClient(VersionItemParam itemParam) {
|
||||
BrokerParam brokerParam = (BrokerParam)itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(brokerParam.getClusterPhyId());
|
||||
|
||||
DescribeLogDirsResult describeLogDirsResult = adminClient.describeLogDirs(Arrays.asList(brokerParam.getBrokerId()), new DescribeLogDirsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
|
||||
return Result.buildSuc(describeLogDirsResult.allDescriptions().get().get(brokerParam.getBrokerId()));
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getBrokerNotExist(brokerParam.getClusterPhyId(), brokerParam.getBrokerId()));
|
||||
} catch (Exception e) {
|
||||
log.error("");
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<Broker>> getBrokersFromZKClient(ClusterPhy clusterPhy) {
|
||||
try {
|
||||
List<Broker> brokerList = new ArrayList<>();
|
||||
|
||||
List<String> brokerIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), BrokerIdsZNode.path(), false);
|
||||
for (String brokerId: brokerIdList) {
|
||||
BrokerMetadata metadata = kafkaZKDAO.getData(clusterPhy.getId(), BrokerIdZNode.path(Integer.valueOf(brokerId)), BrokerMetadata.class);
|
||||
BrokerMetadata.parseAndUpdateBrokerMetadata(metadata);
|
||||
brokerList.add(Broker.buildFrom(clusterPhy.getId(), Integer.valueOf(brokerId), metadata));
|
||||
}
|
||||
|
||||
return Result.buildSuc(brokerList);
|
||||
} catch (Exception e) {
|
||||
log.error("class=BrokerServiceImpl||method=getBrokersFromZKClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<Broker>> getBrokersFromAdminClient(ClusterPhy clusterPhy) {
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhy.getId());
|
||||
|
||||
DescribeClusterResult describeClusterResult = adminClient.describeCluster(new DescribeClusterOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
|
||||
JmxConfig jmxConfig = ConvertUtil.str2ObjByJson(clusterPhy.getJmxProperties(), JmxConfig.class);
|
||||
|
||||
// 当前存活的Broker列表
|
||||
List<Broker> newBrokerList = new ArrayList<>();
|
||||
for (Node node: describeClusterResult.nodes().get()) {
|
||||
newBrokerList.add(this.getStartTimeAndBuildBroker(clusterPhy.getId(), node, jmxConfig));
|
||||
}
|
||||
|
||||
return Result.buildSuc(newBrokerList);
|
||||
} catch (Exception e) {
|
||||
log.error("class=BrokerServiceImpl||method=getBrokersFromAdminClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Broker getStartTimeAndBuildBroker(Long clusterPhyId, Node newNode, JmxConfig jmxConfig) {
|
||||
try {
|
||||
Object object = jmxDAO.getJmxValue(
|
||||
clusterPhyId,
|
||||
newNode.id(),
|
||||
newNode.host(),
|
||||
null,
|
||||
jmxConfig,
|
||||
new ObjectName("java.lang:type=Runtime"),
|
||||
"StartTime"
|
||||
);
|
||||
|
||||
return Broker.buildFrom(clusterPhyId, newNode, object != null? (Long) object: null);
|
||||
} catch (Exception e) {
|
||||
log.error("class=BrokerServiceImpl||method=getStartTimeAndBuildBroker||clusterPhyId={}||brokerNode={}||jmxConfig={}||errMsg=exception!", clusterPhyId, newNode, jmxConfig, e);
|
||||
}
|
||||
|
||||
return Broker.buildFrom(clusterPhyId, newNode, null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.broker.impl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.BrokerSpec;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.config.PlatformClusterConfigPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.config.ConfigGroupEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerSpecService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.config.PlatformClusterConfigService;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
public class BrokerSpecServiceImpl implements BrokerSpecService {
|
||||
|
||||
@Autowired
|
||||
private PlatformClusterConfigService platformClusterConfigService;
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Integer, BrokerSpec> getBrokerSpecMap(Long clusterPhyId) {
|
||||
//获取规格信息
|
||||
Map<String, PlatformClusterConfigPO> specMap = platformClusterConfigService.getByClusterAndGroupWithoutDefault(clusterPhyId, ConfigGroupEnum.BROKER_SPEC.name());
|
||||
if (specMap == null || specMap.isEmpty()){
|
||||
return new HashMap<>();
|
||||
}
|
||||
Map<Integer, BrokerSpec> brokerSpecMap = new HashMap<>();
|
||||
for (Map.Entry<String, PlatformClusterConfigPO> entry : specMap.entrySet()){
|
||||
if (StringUtils.isBlank(entry.getValue().getValue())){
|
||||
continue;
|
||||
}
|
||||
BrokerSpec baseData = ConvertUtil.str2ObjByJson(entry.getValue().getValue(), BrokerSpec.class);
|
||||
brokerSpecMap.put(baseData.getBrokerId(), baseData);
|
||||
}
|
||||
return brokerSpecMap;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.change.record;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.changerecord.KafkaChangeRecordPO;
|
||||
|
||||
|
||||
public interface KafkaChangeRecordService {
|
||||
int insertAndIgnoreDuplicate(KafkaChangeRecordPO recordPO);
|
||||
|
||||
IPage<KafkaChangeRecordPO> pagingByCluster(Long clusterPhyId, PaginationBaseDTO dto);
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.change.record.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.changerecord.KafkaChangeRecordPO;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.change.record.KafkaChangeRecordService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.changerecord.KafkaChangeRecordDAO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class KafkaChangeRecordServiceImpl implements KafkaChangeRecordService {
|
||||
private static final ILog log = LogFactory.getLog(KafkaChangeRecordServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private KafkaChangeRecordDAO kafkaChangeRecordDAO;
|
||||
|
||||
@Override
|
||||
public int insertAndIgnoreDuplicate(KafkaChangeRecordPO recordPO) {
|
||||
try {
|
||||
return kafkaChangeRecordDAO.insert(recordPO);
|
||||
} catch (DuplicateKeyException dke) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public IPage<KafkaChangeRecordPO> pagingByCluster(Long clusterPhyId, PaginationBaseDTO dto) {
|
||||
LambdaQueryWrapper<KafkaChangeRecordPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaChangeRecordPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.orderByDesc(KafkaChangeRecordPO::getUpdateTime);
|
||||
|
||||
return kafkaChangeRecordDAO.selectPage(new Page<>(dto.getPageNo(), dto.getPageSize()), lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsClusterPhyDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public interface ClusterMetricService {
|
||||
|
||||
/**
|
||||
* 从Kafka获取指标
|
||||
*/
|
||||
Result<ClusterMetrics> collectClusterMetricsFromKafka(Long clusterId, String metric);
|
||||
Result<ClusterMetrics> collectClusterMetricsFromKafka(Long clusterId, List<String> metrics);
|
||||
|
||||
/**
|
||||
* listClusterMetricsFromES
|
||||
* @param dto
|
||||
* @return
|
||||
*/
|
||||
Result<List<MetricMultiLinesVO>> listClusterMetricsFromES(MetricsClusterPhyDTO dto);
|
||||
|
||||
/**
|
||||
* 获取多个存储在ES中的指标数据，最新的一条即可
|
||||
*/
|
||||
Result<ClusterMetrics> getLatestMetricsFromES(Long clusterPhyId, List<String> metricNameList);
|
||||
|
||||
/**
|
||||
* 优先从本地缓存获取metrics信息
|
||||
*/
|
||||
ClusterMetrics getLatestMetricsFromCache(Long clusterPhyId);
|
||||
|
||||
Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterId, MetricDTO metricDTO);
|
||||
|
||||
|
||||
PaginationResult<ClusterMetrics> pagingClusterWithLatestMetricsFromES(List<Long> clusterIds,
|
||||
List<SearchTerm> searchMatches,
|
||||
List<SearchShould> shoulds,
|
||||
SearchSort sort, SearchRange range, SearchPage page);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.cluster.ClusterPhyPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.DuplicateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.ParamErrorException;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public interface ClusterPhyService {
|
||||
/**
|
||||
* 先从cache中获取，如果获取不到再从DB获取
|
||||
*/
|
||||
String getVersionFromCacheFirst(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
*/
|
||||
ClusterPhy getClusterByCluster(Long clusterId);
|
||||
|
||||
ClusterPhy getClusterByClusterName(String clusterPhyName);
|
||||
|
||||
/**
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
List<ClusterPhy> listAllClusters();
|
||||
|
||||
/**
|
||||
* 添加集群
|
||||
* @param clusterPhyPO 集群信息
|
||||
* @param operator 操作人
|
||||
* @return
|
||||
* @throws ParamErrorException 参数异常
|
||||
* @throws DuplicateException 数据重复异常
|
||||
* @throws AdminOperateException 操作异常
|
||||
*/
|
||||
Long addClusterPhy(ClusterPhyPO clusterPhyPO, String operator) throws
|
||||
ParamErrorException,
|
||||
DuplicateException,
|
||||
AdminOperateException;
|
||||
|
||||
/**
|
||||
* 移除集群
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param operator 操作人
|
||||
* @return
|
||||
*/
|
||||
Result<Void> removeClusterPhyById(Long clusterPhyId, String operator);
|
||||
|
||||
/**
|
||||
* 修改集群
|
||||
* @param clusterPhyPO 集群信息
|
||||
* @param operator 操作人
|
||||
* @throws ParamErrorException 参数错误
|
||||
* @throws DuplicateException 数据重复异常
|
||||
* @throws NotExistException 集群不存在
|
||||
* @throws AdminOperateException 操作错误
|
||||
*/
|
||||
void modifyClusterPhyById(ClusterPhyPO clusterPhyPO, String operator) throws
|
||||
ParamErrorException,
|
||||
DuplicateException,
|
||||
NotExistException,
|
||||
AdminOperateException;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.util.KafkaValidate;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 22/02/28
|
||||
*/
|
||||
public interface ClusterValidateService {
|
||||
Result<KafkaValidate> checkKafkaLegal(String bootstrapServers, Properties clientProps, String zookeeper);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.ControllerChangeLogPO;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ControllerChangeLogService {
|
||||
|
||||
/**
|
||||
* 添加Controller变化日志
|
||||
* @param controllerChangeLogPO 待添加Controller变化日志对象
|
||||
* @return
|
||||
*/
|
||||
Long addControllerChangeLog(ControllerChangeLogPO controllerChangeLogPO);
|
||||
|
||||
/**
|
||||
* 分页查询结果条件查询出的结果集总数
|
||||
* @param clusterPhyId kafka 集群id
|
||||
* @param dto 封装分页查询条件的对象
|
||||
* @return 分页查询结果条件查询出的结果集总数
|
||||
*/
|
||||
Long queryCount(Long clusterPhyId, PaginationSortDTO dto);
|
||||
|
||||
/**
|
||||
* 分页查询
|
||||
* @param clusterPhyId kafka 集群id
|
||||
* @param dto 封装分页查询条件的对象
|
||||
* @return 分页查询结果集
|
||||
*/
|
||||
List<ControllerChangeLogPO> paginationQuery(Long clusterPhyId, PaginationSortDTO dto);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.google.common.collect.Table;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsClusterPhyDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.ClusterMetricParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ClusterMetricPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterAuthTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.*;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.acl.KafkaAclService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.score.HealthScoreService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.job.JobService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.ClusterMetricESDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||
import org.apache.kafka.common.resource.ResourceType;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.management.InstanceNotFoundException;
|
||||
import javax.management.ObjectName;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics.initWithMetrics;
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.TopicMetricVersionItems.*;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service("clusterMetricService")
|
||||
public class ClusterMetricServiceImpl extends BaseMetricService implements ClusterMetricService {
|
||||
private static final ILog LOGGER = LogFactory.getLog(ClusterMetricServiceImpl.class);
|
||||
|
||||
public static final String CLUSTER_METHOD_DO_NOTHING = "doNothing";
|
||||
|
||||
public static final String CLUSTER_METHOD_GET_TOPIC_SIZE = "getTopicSize";
|
||||
public static final String CLUSTER_METHOD_GET_MESSAGE_SIZE = "getMessageSize";
|
||||
public static final String CLUSTER_METHOD_GET_TOTAL_LOG_SIZE = "getTotalLogSize";
|
||||
public static final String CLUSTER_METHOD_GET_PARTITION_SIZE = "getPartitionSize";
|
||||
public static final String CLUSTER_METHOD_GET_PARTITION_NO_LEADER_SIZE = "getPartitionNoLeaderSize";
|
||||
public static final String CLUSTER_METHOD_GET_HEALTH_SCORE = "getTopicHealthScore";
|
||||
public static final String CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX = "getMetricFromKafkaByTotalBrokersJMX";
|
||||
public static final String CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_CONTROLLER_JMX = "getMetricFromKafkaByControllerJMX";
|
||||
public static final String CLUSTER_METHOD_GET_ZK_COUNT = "getZKCount";
|
||||
public static final String CLUSTER_METHOD_GET_ZK_AVAILABLE = "getZKAvailable";
|
||||
public static final String CLUSTER_METHOD_GET_BROKERS_COUNT = "getBrokersCount";
|
||||
public static final String CLUSTER_METHOD_GET_BROKERS_ALIVE_COUNT = "getBrokersAliveCount";
|
||||
public static final String CLUSTER_METHOD_GET_BROKERS_NOT_ALIVE_COUNT = "getBrokersNotAliveCount";
|
||||
public static final String CLUSTER_METHOD_GET_REPLICAS_COUNT = "getReplicasCount";
|
||||
public static final String CLUSTER_METHOD_GET_GROUP_COUNT = "getGroupsCount";
|
||||
public static final String CLUSTER_METHOD_GET_GROUP_ACTIVE_COUNT = "getGroupActivesCount";
|
||||
public static final String CLUSTER_METHOD_GET_GROUP_EMPTY_COUNT = "getGroupEmptyCount";
|
||||
public static final String CLUSTER_METHOD_GET_GROUP_REBALANCED_COUNT = "getGroupRebalancedCount";
|
||||
public static final String CLUSTER_METHOD_GET_GROUP_DEAD_COUNT = "getGroupDeadCount";
|
||||
public static final String CLUSTER_METHOD_GET_ALIVE = "isClusterAlive";
|
||||
|
||||
public static final String CLUSTER_METHOD_GET_ACL_ENABLE = "getAclEnable";
|
||||
public static final String CLUSTER_METHOD_GET_ACLS = "getAcls";
|
||||
public static final String CLUSTER_METHOD_GET_ACL_USERS = "getAclUsers";
|
||||
public static final String CLUSTER_METHOD_GET_ACL_TOPICS = "getAclTopics";
|
||||
public static final String CLUSTER_METHOD_GET_ACL_GROUPS = "getAclGroups";
|
||||
|
||||
public static final String CLUSTER_METHOD_GET_JOBS = "getJobs";
|
||||
public static final String CLUSTER_METHOD_GET_JOBS_RUNNING = "getJobsRunning";
|
||||
public static final String CLUSTER_METHOD_GET_JOBS_WAITING = "getJobsWaiting";
|
||||
public static final String CLUSTER_METHOD_GET_JOBS_SUCCESS = "getJobsSuccess";
|
||||
public static final String CLUSTER_METHOD_GET_JOBS_FAILED = "getJobsFailed";
|
||||
|
||||
@Autowired
|
||||
private HealthScoreService healthScoreService;
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private BrokerMetricService brokerMetricService;
|
||||
|
||||
@Autowired
|
||||
private TopicMetricService topicMetricService;
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private GroupService groupService;
|
||||
|
||||
@Autowired
|
||||
private KafkaJMXClient kafkaJMXClient;
|
||||
|
||||
@Autowired
|
||||
private ClusterMetricESDAO clusterMetricESDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaControllerService kafkaControllerService;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAclService kafkaAclService;
|
||||
|
||||
@Autowired
|
||||
private JobService jobService;
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyService clusterPhyService;
|
||||
|
||||
private final Cache<Long, ClusterMetrics> clusterLatestMetricsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(180, TimeUnit.SECONDS)
|
||||
.maximumSize(1000)
|
||||
.build();
|
||||
|
||||
@PostConstruct
|
||||
@Scheduled(cron = "0 0/1 * * * ?")
|
||||
private void flushClusterLatestMetricsCache() {
|
||||
for (ClusterPhy clusterPhy: clusterPhyService.listAllClusters()) {
|
||||
FutureUtil.quickStartupFutureUtil.submitTask(() -> this.updateCacheAndGetMetrics(clusterPhy.getId()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.METRIC_CLUSTER;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<String> listMetricPOFields(){
|
||||
return BeanUtil.listBeanFields(ClusterMetricPO.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initRegisterVCHandler(){
|
||||
registerVCHandler( CLUSTER_METHOD_DO_NOTHING, this::doNothing);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_TOPIC_SIZE, this::getTopicSize);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_MESSAGE_SIZE, this::getMessageSize);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_TOTAL_LOG_SIZE, this::getTotalLogSize);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_PARTITION_SIZE, this::getPartitionSize);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_PARTITION_NO_LEADER_SIZE, this::getPartitionNoLeaderSize);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_HEALTH_SCORE, this::getTopicHealthScore);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX, this::getMetricFromKafkaByTotalBrokersJMX);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_CONTROLLER_JMX, this::getMetricFromKafkaByControllerJMX);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ZK_COUNT, this::getZKCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ZK_AVAILABLE, this::getZKAvailable);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_BROKERS_COUNT, this::getBrokersCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_BROKERS_ALIVE_COUNT, this::getBrokersAliveCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_BROKERS_NOT_ALIVE_COUNT, this::getBrokersNotAliveCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_REPLICAS_COUNT, this::getReplicasCount);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_GROUP_COUNT, this::getGroupsCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_GROUP_ACTIVE_COUNT, this::getGroupActivesCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_GROUP_EMPTY_COUNT, this::getGroupEmptyCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_GROUP_REBALANCED_COUNT, this::getGroupRebalancedCount);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_GROUP_DEAD_COUNT, this::getGroupDeadCount);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ALIVE, this::isClusterAlive);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ACL_ENABLE, this::getAclEnable);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ACLS, this::getAcls);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ACL_USERS, this::getAclUsers);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ACL_TOPICS, this::getAclTopics);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_ACL_GROUPS, this::getAclGroups);
|
||||
|
||||
registerVCHandler( CLUSTER_METHOD_GET_JOBS, this::getJobs);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_JOBS_RUNNING, this::getJobsRunning);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_JOBS_WAITING, this::getJobsWaiting);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_JOBS_SUCCESS, this::getJobsSuccess);
|
||||
registerVCHandler( CLUSTER_METHOD_GET_JOBS_FAILED, this::getJobsFailed);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ClusterMetrics> collectClusterMetricsFromKafka(Long clusterId, String metric){
|
||||
try {
|
||||
ClusterMetricParam clusterMetricParam = new ClusterMetricParam(clusterId, metric );
|
||||
return (Result<ClusterMetrics>)doVCHandler(clusterId, metric, clusterMetricParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ClusterMetrics> collectClusterMetricsFromKafka(Long clusterId, List<String> metrics){
|
||||
ClusterMetrics clusterMetrics = new ClusterMetrics();
|
||||
clusterMetrics.setClusterPhyId(clusterId);
|
||||
|
||||
for(String metric :metrics){
|
||||
if(null != clusterMetrics.getMetrics().get(metric)){continue;}
|
||||
|
||||
Result<ClusterMetrics> ret = collectClusterMetricsFromKafka(clusterId, metric);
|
||||
if(null != ret && ret.successful()){
|
||||
clusterMetrics.putMetric(ret.getData().getMetrics());
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(clusterMetrics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricMultiLinesVO>> listClusterMetricsFromES(MetricsClusterPhyDTO dto) {
|
||||
Long startTime = dto.getStartTime();
|
||||
Long endTime = dto.getEndTime();
|
||||
Integer topN = dto.getTopNu();
|
||||
String aggType = dto.getAggType();
|
||||
List<Long> clusterPhyIds = dto.getClusterPhyIds();
|
||||
List<String> metrics = dto.getMetricsNames();
|
||||
|
||||
try {
|
||||
Table<String/*metric*/, Long/*clusterId*/, List<MetricPointVO>> retTable;
|
||||
if(CollectionUtils.isEmpty(clusterPhyIds)) {
|
||||
return Result.buildFailure( "clusterPhyIds is empty!" );
|
||||
}else {
|
||||
retTable = clusterMetricESDAO.listClusterMetricsByClusterIds(metrics, aggType, clusterPhyIds, startTime, endTime);
|
||||
}
|
||||
|
||||
List<MetricMultiLinesVO> multiLinesVOS = metricMap2VO(clusterPhyIds.get(0), retTable.rowMap());
|
||||
return Result.buildSuc(multiLinesVOS);
|
||||
}catch (Exception e){
|
||||
LOGGER.error("method=listClusterMetricsFromES||clusters={}||metrics={}||topN={}||aggType={}",
|
||||
clusterPhyIds, metrics, topN, aggType);
|
||||
|
||||
return Result.buildFailure(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ClusterMetrics> getLatestMetricsFromES(Long clusterPhyId, List<String> metricNames) {
|
||||
ClusterMetricPO clusterMetricPO = clusterMetricESDAO.getClusterLatestMetrics(clusterPhyId, metricNames);
|
||||
return Result.buildSuc(ConvertUtil.obj2Obj(clusterMetricPO, ClusterMetrics.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterMetrics getLatestMetricsFromCache(Long clusterPhyId) {
|
||||
ClusterMetrics metrics = clusterLatestMetricsCache.getIfPresent(clusterPhyId);
|
||||
if (metrics != null) {
|
||||
return metrics;
|
||||
}
|
||||
|
||||
return new ClusterMetrics(clusterPhyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterId, MetricDTO dto) {
|
||||
Map<String/*metric*/, MetricPointVO> metricPointMap = clusterMetricESDAO.getClusterMetricsPoint(clusterId,
|
||||
dto.getMetricsNames(), dto.getAggType(), dto.getStartTime(), dto.getEndTime());
|
||||
|
||||
List<MetricPointVO> metricPoints = new ArrayList<>(metricPointMap.values());
|
||||
return Result.buildSuc(metricPoints);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<ClusterMetrics> pagingClusterWithLatestMetricsFromES(List<Long> clusterIds,
|
||||
List<SearchTerm> terms,
|
||||
List<SearchShould> shoulds,
|
||||
SearchSort sort,
|
||||
SearchRange range,
|
||||
SearchPage page){
|
||||
setQueryMetricFlag(sort);
|
||||
setQueryMetricFlag(range);
|
||||
setQueryMetricFlag(terms);
|
||||
setQueryMetricFlag(shoulds);
|
||||
|
||||
List<ClusterMetricPO> clusterMetricPOS = clusterMetricESDAO.pagingClusterWithLatestMetrics(terms, shoulds, sort, range);
|
||||
|
||||
List<ClusterMetricPO> filterClusterMetricPOs = clusterMetricPOS.stream()
|
||||
.filter(c -> clusterIds.contains(c.getClusterPhyId()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if(null == page){
|
||||
return PaginationResult.buildSuc(ConvertUtil.list2List(filterClusterMetricPOs, ClusterMetrics.class), clusterIds.size(),
|
||||
1, 10);
|
||||
}
|
||||
|
||||
int startIdx = Math.min((page.getPageNo() - 1) * page.getPageSize(), filterClusterMetricPOs.size());
|
||||
int endIdx = Math.min(startIdx +page.getPageSize(), filterClusterMetricPOs.size());
|
||||
|
||||
List<ClusterMetricPO> subClusterMetricPOSs = filterClusterMetricPOs.subList(startIdx, endIdx);
|
||||
|
||||
return PaginationResult.buildSuc(ConvertUtil.list2List(subClusterMetricPOSs, ClusterMetrics.class), clusterIds.size(),
|
||||
page.getPageNo(), page.getPageSize());
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private ClusterMetrics updateCacheAndGetMetrics(Long clusterPhyId) {
|
||||
try {
|
||||
Result<ClusterMetrics> metricsResult = this.getLatestMetricsFromES(clusterPhyId, Arrays.asList());
|
||||
if (metricsResult.hasData()) {
|
||||
LOGGER.info("method=updateCacheAndGetMetrics||clusterPhyId={}||msg=success", clusterPhyId);
|
||||
clusterLatestMetricsCache.put(clusterPhyId, metricsResult.getData());
|
||||
|
||||
return metricsResult.getData();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=updateCacheAndGetMetrics||clusterPhyId={}||errMsg=exception!", clusterPhyId, e);
|
||||
}
|
||||
|
||||
ClusterMetrics clusterMetrics = new ClusterMetrics(clusterPhyId);
|
||||
clusterLatestMetricsCache.put(clusterPhyId, clusterMetrics);
|
||||
return clusterMetrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* doNothing
|
||||
*/
|
||||
private Result<ClusterMetrics> doNothing(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
return Result.buildSuc(initWithMetrics(param.getClusterId(), param.getMetric(), -1.0f));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取集群的健康分
|
||||
*/
|
||||
private Result<ClusterMetrics> getTopicHealthScore(VersionItemParam metricParam){
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
ClusterMetrics clusterMetrics = healthScoreService.calClusterHealthScore(param.getClusterId());
|
||||
return Result.buildSuc(clusterMetrics);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取集群的 totalLogSize
|
||||
* @param metricParam
|
||||
* @return
|
||||
*/
|
||||
private Result<ClusterMetrics> getTotalLogSize(VersionItemParam metricParam){
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
return getMetricFromKafkaByTotalTopics(param.getClusterId(), param.getMetric(), TOPIC_METRIC_LOG_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取集群的 messageSize
|
||||
*/
|
||||
private Result<ClusterMetrics> getMessageSize(VersionItemParam metricParam){
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
return getMetricFromKafkaByTotalTopics(param.getClusterId(), param.getMetric(), TOPIC_METRIC_MESSAGES);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取集群的Topic个数
|
||||
*/
|
||||
private Result<ClusterMetrics> getTopicSize(VersionItemParam metricParam){
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer topicNu = topicService.getTopicSizeFromCacheFirst(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, topicNu.floatValue()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取集群的Partition个数
|
||||
*/
|
||||
private Result<ClusterMetrics> getPartitionSize(VersionItemParam metricParam){
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer partitionNu = partitionService.getPartitionSizeByClusterId(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, partitionNu.floatValue()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取集群无Leader的partition个数
|
||||
*/
|
||||
private Result<ClusterMetrics> getPartitionNoLeaderSize(VersionItemParam metricParam){
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer noLeaders = partitionService.getNoLeaderPartitionSizeByClusterId(clusterId);
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, noLeaders.floatValue()));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getZKCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(clusterId);
|
||||
if(null == clusterPhy || StringUtils.isEmpty(clusterPhy.getZookeeper())) {
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, 0f));
|
||||
}
|
||||
|
||||
String zookeepers = clusterPhy.getZookeeper();
|
||||
String[] zookeeper = zookeepers.split(",");
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, zookeeper.length));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getZKAvailable(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
try {
|
||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(clusterId);
|
||||
if (clusterPhy == null) {
|
||||
// 集群未加载，或者不存在，当前信息未知
|
||||
return Result.buildFromRSAndMsg(NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterId));
|
||||
}
|
||||
|
||||
if (ValidateUtils.isBlank(clusterPhy.getZookeeper())) {
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, Constant.INVALID_CODE));
|
||||
}
|
||||
|
||||
kafkaAdminZKClient.getClient(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, Constant.YES));
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, Constant.NO));
|
||||
}
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getBrokersCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
List<Broker> brokers = brokerService.listAllBrokersFromDB(clusterId);
|
||||
int brokerNu = (CollectionUtils.isEmpty(brokers)) ? 0 : brokers.size();
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, brokerNu));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getBrokersAliveCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
List<Broker> brokers = brokerService.listAliveBrokersFromDB(clusterId);
|
||||
int brokerNu = (CollectionUtils.isEmpty(brokers)) ? 0 : brokers.size();
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, brokerNu));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getBrokersNotAliveCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
List<Broker> brokers = brokerService.listNotAliveBrokersFromDB(clusterId);
|
||||
int brokerNu = (CollectionUtils.isEmpty(brokers)) ? 0 : brokers.size();
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, brokerNu));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getReplicasCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
Integer replicasCount = 0;
|
||||
List<Topic> topicList = topicService.listTopicsFromCacheFirst(param.getClusterId());
|
||||
for (Topic topic: topicList) {
|
||||
replicasCount += (topic.getPartitionNum() * topic.getReplicaNum());
|
||||
}
|
||||
|
||||
return Result.buildSuc(initWithMetrics(param.getClusterId(), param.getMetric(), replicasCount.floatValue()));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getGroupsCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = groupService.calGroupCount(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getGroupActivesCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = groupService.calGroupStatCount(clusterId, GroupStateEnum.ACTIVE);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getGroupEmptyCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = groupService.calGroupStatCount(clusterId, GroupStateEnum.EMPTY);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getGroupRebalancedCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = groupService.calGroupStatCount(clusterId, GroupStateEnum.PREPARING_RE_BALANCE);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getGroupDeadCount(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = groupService.calGroupStatCount(clusterId, GroupStateEnum.DEAD);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> isClusterAlive(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
KafkaController controller = kafkaControllerService.getKafkaControllerFromDB(clusterId);
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, (null != controller) ? 1 : 0));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getAclEnable(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(clusterId);
|
||||
if (clusterPhy == null) {
|
||||
return Result.buildFromRSAndMsg(NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterId));
|
||||
}
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, ClusterAuthTypeEnum.enableAuth(clusterPhy.getAuthType())? Constant.YES: Constant.NO));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getAcls(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = kafkaAclService.countKafkaAclFromDB(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getAclUsers(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = kafkaAclService.countKafkaUserAndDistinctFromDB(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getAclTopics(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = kafkaAclService.countResTypeAndDistinctFromDB(clusterId, ResourceType.TOPIC);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getAclGroups(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = kafkaAclService.countResTypeAndDistinctFromDB(clusterId, ResourceType.GROUP);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getJobs(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = jobService.countJobsByCluster(clusterId);
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getJobsRunning(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = jobService.countJobsByClusterAndJobStatus(clusterId, JobStatusEnum.RUNNING.getStatus());
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getJobsWaiting(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = jobService.countJobsByClusterAndJobStatus(clusterId, JobStatusEnum.WAITING.getStatus());
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getJobsSuccess(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = jobService.countJobsByClusterAndJobStatus(clusterId, JobStatusEnum.SUCCESS.getStatus());
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
private Result<ClusterMetrics> getJobsFailed(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
Integer count = jobService.countJobsByClusterAndJobStatus(clusterId, JobStatusEnum.FAILED.getStatus());
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, count));
|
||||
}
|
||||
|
||||
/**
|
||||
* 从某一个 controller 的 JMX 中获取指标再聚合得到集群的指标
|
||||
* @param metricParam
|
||||
* @return
|
||||
*/
|
||||
private Result<ClusterMetrics> getMetricFromKafkaByControllerJMX(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
//2、获取集群 controller 信息
|
||||
KafkaController controller = kafkaControllerService.getKafkaControllerFromDB(clusterId);
|
||||
if(null == controller){return Result.buildFailure(CONTROLLER_NOT_EXIST);}
|
||||
|
||||
//3、获取jmx连接
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, controller.getBrokerId());
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
|
||||
//3、获取jmx指标
|
||||
try {
|
||||
String value = jmxConnectorWrap.getAttribute(new ObjectName(jmxInfo.getJmxObjectName()),
|
||||
jmxInfo.getJmxAttribute()).toString();
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, Float.valueOf(value)));
|
||||
} catch (InstanceNotFoundException e) {
|
||||
return Result.buildFailure(VC_JMX_INSTANCE_NOT_FOUND);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=getMetricFromKafkaByControllerJMX||cluster={}||brokerId={}||metrics={}||jmx={}||msg={}",
|
||||
clusterId, controller.getBrokerId(), metric, jmxInfo.getJmxObjectName(), e.getClass().getName());
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从所有broker的 JMX 中获取指标再聚合得到集群的指标
|
||||
*/
|
||||
private Result<ClusterMetrics> getMetricFromKafkaByTotalBrokersJMX(VersionItemParam metricParam) {
|
||||
ClusterMetricParam param = (ClusterMetricParam)metricParam;
|
||||
|
||||
String metric = param.getMetric();
|
||||
Long clusterId = param.getClusterId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
List<Broker> brokers = brokerService.listAliveBrokersFromDB(clusterId);
|
||||
|
||||
float metricVale = 0f;
|
||||
for(Broker broker : brokers){
|
||||
Result<BrokerMetrics> ret = brokerMetricService.collectBrokerMetricsFromKafkaWithCacheFirst(clusterId, broker.getBrokerId(), metric);
|
||||
|
||||
if(null == ret || ret.failed() || null == ret.getData()){continue;}
|
||||
|
||||
BrokerMetrics brokerMetrics = ret.getData();
|
||||
if(null != brokerMetrics && null != brokerMetrics.getMetrics().get(metric)){
|
||||
metricVale += Double.valueOf(brokerMetrics.getMetrics().get(metric));
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, metricVale));
|
||||
}
|
||||
|
||||
/**
|
||||
* 从所有的 Topic 的指标中加总聚合得到集群的指标
|
||||
*/
|
||||
private Result<ClusterMetrics> getMetricFromKafkaByTotalTopics(Long clusterId, String metric, String topicMetric){
|
||||
List<Topic> topics = topicService.listTopicsFromCacheFirst(clusterId);
|
||||
|
||||
float metricsSum = 0f;
|
||||
for(Topic topic : topics){
|
||||
Result<List<TopicMetrics>> ret = topicMetricService.collectTopicMetricsFromKafkaWithCacheFirst(
|
||||
clusterId,
|
||||
topic.getTopicName(),
|
||||
topicMetric
|
||||
);
|
||||
|
||||
if(null == ret || ret.failed() || CollectionUtils.isEmpty(ret.getData())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<TopicMetrics> topicMetrics = ret.getData();
|
||||
for (TopicMetrics metrics : topicMetrics) {
|
||||
if(metrics.isBBrokerAgg()){
|
||||
metricsSum += Double.valueOf(metrics.getMetrics().get(topicMetric));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(initWithMetrics(clusterId, metric, metricsSum));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.cluster.ClusterPhyPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.DuplicateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.ParamErrorException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.cluster.ClusterPhyDAO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service
|
||||
public class ClusterPhyServiceImpl implements ClusterPhyService {
|
||||
private static final ILog log = LogFactory.getLog(ClusterPhyServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyDAO clusterPhyDAO;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Override
|
||||
public String getVersionFromCacheFirst(Long clusterPhyId) {
|
||||
ClusterPhy clusterPhy = LoadedClusterPhyCache.getByPhyId(clusterPhyId);
|
||||
if (clusterPhy != null) {
|
||||
return clusterPhy.getKafkaVersion();
|
||||
}
|
||||
|
||||
ClusterPhyPO clusterPhyPO = clusterPhyDAO.selectById(clusterPhyId);
|
||||
if(null == clusterPhyPO) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return clusterPhyPO.getKafkaVersion();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterPhy getClusterByCluster(Long clusterId) {
|
||||
return ConvertUtil.obj2Obj(clusterPhyDAO.selectById(clusterId), ClusterPhy.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterPhy getClusterByClusterName(String clusterPhyName) {
|
||||
LambdaQueryWrapper<ClusterPhyPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(ClusterPhyPO::getName, clusterPhyName);
|
||||
ClusterPhyPO phyPO = clusterPhyDAO.selectOne(lambdaQueryWrapper);
|
||||
if (phyPO == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return ConvertUtil.obj2Obj(phyPO, ClusterPhy.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ClusterPhy> listAllClusters() {
|
||||
List<ClusterPhyPO> clusterPhyPOS = clusterPhyDAO.selectList(null);
|
||||
return ConvertUtil.list2List(clusterPhyPOS, ClusterPhy.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long addClusterPhy(ClusterPhyPO clusterPhyPO, String operator) throws ParamErrorException, DuplicateException, AdminOperateException {
|
||||
log.info("method=addClusterPhy||clusterPhyPO={}||operator={}||msg=add cluster start", clusterPhyPO, operator);
|
||||
|
||||
// 检查字段
|
||||
Result<Void> rv = ClusterPhyPO.checkFieldAddToDBLegal(clusterPhyPO, operator);
|
||||
if (rv.failed()) {
|
||||
log.warn("method=addClusterPhy||clusterPhyPO={}||operator={}||result={}||errMsg=add cluster failed", clusterPhyPO, operator, rv);
|
||||
|
||||
// 字段非法，则直接抛出异常
|
||||
throw new ParamErrorException(rv.getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
clusterPhyDAO.addAndSetId(clusterPhyPO);
|
||||
|
||||
// 记录操作
|
||||
OplogDTO oplogDTO = new OplogDTO(operator,
|
||||
OperationEnum.ADD.getDesc(),
|
||||
ModuleEnum.KAFKA_CLUSTER.getDesc(),
|
||||
MsgConstant.getClusterBizStr(clusterPhyPO.getId(), clusterPhyPO.getName()),
|
||||
String.format("新增集群:%s",clusterPhyPO.toString()));
|
||||
opLogWrapService.saveOplogAndIgnoreException(oplogDTO);
|
||||
|
||||
log.info("method=addClusterPhy||clusterPhyId={}||operator={}||msg=add cluster finished", clusterPhyPO.getId(), operator);
|
||||
|
||||
return clusterPhyPO.getId();
|
||||
} catch (DuplicateKeyException dke) {
|
||||
log.warn("method=addClusterPhy||clusterPhyId={}||operator={}||msg=add cluster failed||errMsg=duplicate data", clusterPhyPO.getId(), operator);
|
||||
|
||||
throw new DuplicateException(String.format("clusterName:%s duplicated", clusterPhyPO.getName()));
|
||||
} catch (Exception e) {
|
||||
log.error("cmethod=addClusterPhy||clusterPhyId={}||operator={}||msg=add cluster failed||errMsg=exception!", clusterPhyPO.getId(), operator, e);
|
||||
|
||||
throw new AdminOperateException("add cluster failed", e, ResultStatus.MYSQL_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> removeClusterPhyById(Long clusterPhyId, String operator) {
|
||||
log.info("method=removeClusterPhyById||clusterPhyId={}||operator={}||msg=remove cluster start",
|
||||
clusterPhyId, operator);
|
||||
|
||||
try {
|
||||
ClusterPhy clusterPhy = this.getClusterByCluster(clusterPhyId);
|
||||
if (clusterPhy == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(clusterPhyId));
|
||||
}
|
||||
|
||||
clusterPhyDAO.deleteById(clusterPhyId);
|
||||
|
||||
log.info("method=removeClusterPhyById||clusterPhyId={}||operator={}||msg=remove cluster finished",
|
||||
clusterPhyId, operator);
|
||||
|
||||
// 记录操作
|
||||
OplogDTO oplogDTO = new OplogDTO(operator,
|
||||
OperationEnum.DELETE.getDesc(),
|
||||
ModuleEnum.KAFKA_CLUSTER.getDesc(),
|
||||
MsgConstant.getClusterBizStr(clusterPhy.getId(), clusterPhy.getName()),
|
||||
String.format("删除集群:%s",clusterPhy.toString()));
|
||||
opLogWrapService.saveOplogAndIgnoreException(oplogDTO);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=removeClusterPhyById||clusterPhyId={}||operator={}||msg=remove cluster failed||errMsg=exception!",
|
||||
clusterPhyId, operator, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void modifyClusterPhyById(ClusterPhyPO clusterPhyPO, String operator) throws
|
||||
ParamErrorException,
|
||||
DuplicateException,
|
||||
NotExistException,
|
||||
AdminOperateException {
|
||||
log.info("method=modifyClusterPhyById||clusterPhyPO={}||operator={}||msg=modify cluster start",
|
||||
clusterPhyPO, operator);
|
||||
|
||||
// 检查字段
|
||||
Result<Void> rv = ClusterPhyPO.checkFieldModifyToDBLegal(clusterPhyPO, operator);
|
||||
if (rv.failed()) {
|
||||
log.warn("method=modifyClusterPhyById||clusterPhyPO={}||result={}||operator={}||msg=modify cluster failed||errMsg=param illegal",
|
||||
clusterPhyPO, rv, operator);
|
||||
|
||||
// 字段非法，则直接抛出异常
|
||||
throw new ParamErrorException(rv.getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
int affectRow = clusterPhyDAO.updateById(clusterPhyPO);
|
||||
if (affectRow <= 0) {
|
||||
throw new NotExistException(MsgConstant.getClusterPhyNotExist(clusterPhyPO.getId()));
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.KAFKA_CLUSTER.getDesc(),
|
||||
MsgConstant.getClusterBizStr(clusterPhyPO.getId(), clusterPhyPO.getName()),
|
||||
String.format("修改集群, 详情参数信息:%s", clusterPhyPO.toString()))
|
||||
);
|
||||
|
||||
log.info("method=modifyClusterPhyById||clusterPhyId={}||operator={}||msg=modify cluster finished",
|
||||
clusterPhyPO.getId(), operator);
|
||||
} catch (DuplicateKeyException dke) {
|
||||
log.warn("method=modifyClusterPhyById||clusterPhyPO={}||operator={}||msg=modify cluster failed||errMsg=duplicate clusterName",
|
||||
clusterPhyPO, operator);
|
||||
|
||||
throw new DuplicateException(String.format("clusterName:%s duplicated", clusterPhyPO.getName()));
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=modifyClusterPhyById||clusterPhyPO={}||operator={}||msg=modify cluster failed||errMsg=cluster not exist",
|
||||
clusterPhyPO, operator);
|
||||
|
||||
throw nee;
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyClusterPhyById||clusterPhyPO={}||operator={}||msg=modify cluster failed||errMsg=exception!",
|
||||
clusterPhyPO, operator, e);
|
||||
|
||||
throw new AdminOperateException("modify cluster failed", e, ResultStatus.MYSQL_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.JmxConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.util.KafkaValidate;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.valid.ValidateKafkaAddressErrorEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterValidateService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.jmx.JmxDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.impl.KafkaZKDAOImpl;
|
||||
import kafka.server.KafkaConfig;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.clients.consumer.ConsumerConfig;
|
||||
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.config.ConfigResource;
|
||||
import org.apache.kafka.common.internals.Topic;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.management.ObjectName;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 22/02/28
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ClusterValidateServiceImpl implements ClusterValidateService {
|
||||
private static final ILog logger = LogFactory.getLog(KafkaZKDAOImpl.class);
|
||||
|
||||
@Autowired
|
||||
private JmxDAO jmxDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Override
|
||||
public Result<KafkaValidate> checkKafkaLegal(String bootstrapServers, Properties clientProps, String zookeeper) {
|
||||
if (ValidateUtils.isBlank(bootstrapServers) && ValidateUtils.isBlank(zookeeper)) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "bootstrapServers与zookeeper不能同时为空");
|
||||
}
|
||||
|
||||
if (!ValidateUtils.isBlank(bootstrapServers) && !ValidateUtils.isBlank(zookeeper)) {
|
||||
// 检查ZK 和 BS
|
||||
return this.checkZKAndBSLegal(bootstrapServers, clientProps, zookeeper);
|
||||
}
|
||||
|
||||
if (!ValidateUtils.isBlank(zookeeper)) {
|
||||
// 仅检查ZK
|
||||
return Result.buildSuc(this.checkZKLegalAndTryGetInfo(zookeeper));
|
||||
}
|
||||
|
||||
// 检查BS
|
||||
if (!checkBSLegal(bootstrapServers, clientProps)) {
|
||||
// BS 非法，则直接返回
|
||||
KafkaValidate kafkaValidate = new KafkaValidate();
|
||||
kafkaValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.BS_ERROR, ValidateKafkaAddressErrorEnum.BS_CONNECT_FAILED));
|
||||
return Result.buildSuc(kafkaValidate);
|
||||
}
|
||||
|
||||
// BS合法，需要尝试获取其他信息
|
||||
return Result.buildSuc(this.getDataAndIgnoreCheckBSLegal(bootstrapServers, clientProps));
|
||||
}
|
||||
|
||||
private Result<KafkaValidate> checkZKAndBSLegal(String bootstrapServers, Properties clientProps, String zookeeper) {
|
||||
KafkaValidate kafkaValidate = this.checkZKLegalAndTryGetInfo(zookeeper);
|
||||
if (!checkBSLegal(bootstrapServers, clientProps)) {
|
||||
kafkaValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.BS_ERROR, ValidateKafkaAddressErrorEnum.BS_CONNECT_FAILED));
|
||||
}
|
||||
|
||||
return Result.buildSuc(kafkaValidate);
|
||||
}
|
||||
|
||||
private boolean checkBSLegal(String bootstrapServers, Properties clientProps) {
|
||||
clientProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
|
||||
clientProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
|
||||
clientProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
|
||||
|
||||
// 判断地址是否有问题
|
||||
KafkaConsumer<String, String> kafkaConsumer = null;
|
||||
try {
|
||||
kafkaConsumer = new KafkaConsumer<>(clientProps);
|
||||
kafkaConsumer.partitionsFor(Topic.GROUP_METADATA_TOPIC_NAME, Duration.ofMillis(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
} catch (Exception e) {
|
||||
logger.error("method=checkBootstrapServersLegal||bootstrapServers={}||clientProps={}||errMsg=exception", bootstrapServers, clientProps, e);
|
||||
|
||||
return false;
|
||||
} finally {
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.close();
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private KafkaValidate getDataAndIgnoreCheckBSLegal(String bootstrapServers, Properties clientProps) {
|
||||
AdminClient adminClient = null;
|
||||
|
||||
KafkaValidate kafkaBSValidate = new KafkaValidate();
|
||||
try {
|
||||
adminClient = KafkaAdminClient.create(clientProps);
|
||||
|
||||
// 获取集群broker节点
|
||||
DescribeClusterResult describeClusterResult = adminClient.describeCluster(new DescribeClusterOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
Collection<Node> nodeCollection = describeClusterResult.nodes().get();
|
||||
if (nodeCollection.isEmpty()) {
|
||||
kafkaBSValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.BS_ERROR, ValidateKafkaAddressErrorEnum.BS_CONNECT_FAILED));
|
||||
return kafkaBSValidate;
|
||||
}
|
||||
|
||||
// 获取Broker中的ZK配置
|
||||
ConfigResource configResource = new ConfigResource(ConfigResource.Type.BROKER, String.valueOf(new ArrayList<>(nodeCollection).get(0).id()));
|
||||
DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(Arrays.asList(configResource), new DescribeConfigsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
Map<ConfigResource, Config> configMap = describeConfigsResult.all().get();
|
||||
|
||||
ConfigEntry configEntry = configMap.get(configResource).get(KafkaConfig.ZkConnectProp());
|
||||
if (configEntry == null || configEntry.value() == null) {
|
||||
kafkaBSValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.BS_ERROR, ValidateKafkaAddressErrorEnum.BS_CONNECT_SUCCESS_BUT_RUN_RAFT));
|
||||
return kafkaBSValidate;
|
||||
}
|
||||
|
||||
return this.checkZKLegalAndTryGetInfo(configEntry.value().replace(" ", ""));
|
||||
} catch (Exception e) {
|
||||
logger.error("method=checkBootstrapServersLegal||bootstrapServers={}||clientProps={}||msg=get cluster-info failed||errMsg=exception", bootstrapServers, clientProps, e);
|
||||
|
||||
kafkaBSValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.BS_ERROR, ValidateKafkaAddressErrorEnum.BS_CONNECT_SUCCESS_BUT_API_NOT_SUPPORT));
|
||||
return kafkaBSValidate;
|
||||
} finally {
|
||||
if (adminClient != null) {
|
||||
adminClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private KafkaValidate checkZKLegalAndTryGetInfo(String zookeeper) {
|
||||
KafkaValidate kafkaZKValidate = new KafkaValidate();
|
||||
kafkaZKValidate.setZookeeper(zookeeper);
|
||||
|
||||
Broker broker = null;
|
||||
try {
|
||||
broker = kafkaZKDAO.getBrokerMetadata(zookeeper);
|
||||
if (broker == null) {
|
||||
kafkaZKValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.ZK_ERROR, ValidateKafkaAddressErrorEnum.ZK_CONNECT_SUCCESS_BUT_NODE_NOT_EXIST));
|
||||
} else {
|
||||
kafkaZKValidate.setJmxPort(broker.getJmxPort());
|
||||
}
|
||||
} catch (KeeperException.NoNodeException nne) {
|
||||
logger.warn("method=checkZKLegalAndTryGetInfo||zookeeper={}||errMsg=NoNodeException", zookeeper);
|
||||
|
||||
// ZK正常，但是节点不存在
|
||||
kafkaZKValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.ZK_ERROR, ValidateKafkaAddressErrorEnum.ZK_CONNECT_SUCCESS_BUT_NODE_NOT_EXIST));
|
||||
} catch (Exception e) {
|
||||
logger.error("method=checkZKLegalAndTryGetInfo||zookeeper={}||errMsg={}", zookeeper, e.getMessage());
|
||||
|
||||
kafkaZKValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.ZK_ERROR, ValidateKafkaAddressErrorEnum.ZK_CONNECT_ERROR));
|
||||
}
|
||||
|
||||
if (broker == null) {
|
||||
return kafkaZKValidate;
|
||||
}
|
||||
|
||||
kafkaZKValidate.setJmxPort(broker.getJmxPort());
|
||||
if (broker.getJmxPort() == null || Constant.INVALID_CODE == broker.getJmxPort()) {
|
||||
kafkaZKValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.JMX_ERROR, ValidateKafkaAddressErrorEnum.JMX_WITHOUT_OPEN));
|
||||
return kafkaZKValidate;
|
||||
}
|
||||
|
||||
try {
|
||||
Object jmxValue = jmxDAO.getJmxValue(broker.getHost(), broker.getJmxPort(), new JmxConfig(), new ObjectName(String.format("kafka.server:type=app-info,id=%d", broker.getBrokerId())), "Version");
|
||||
if (jmxValue instanceof String) {
|
||||
kafkaZKValidate.setKafkaVersion((String) jmxValue);
|
||||
} else {
|
||||
kafkaZKValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.JMX_ERROR, ValidateKafkaAddressErrorEnum.CONNECT_JMX_ERROR));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("method=checkZKLegalAndTryGetInfo||zookeeper={}||broker={}||errMsg={}", zookeeper, broker, e.getMessage());
|
||||
|
||||
kafkaZKValidate.getErrList().add(Result.buildFrom(ValidateKafkaAddressErrorEnum.JMX_ERROR, ValidateKafkaAddressErrorEnum.CONNECT_JMX_ERROR));
|
||||
}
|
||||
|
||||
|
||||
return kafkaZKValidate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.cluster.impl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationSortDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.ControllerChangeLogPO;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ControllerChangeLogService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.ControllerChangeLogDAO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ControllerChangeLogServiceImpl implements ControllerChangeLogService {
|
||||
|
||||
@Autowired
|
||||
private ControllerChangeLogDAO controllerChangeLogDAO;
|
||||
|
||||
@Override
|
||||
public Long addControllerChangeLog(ControllerChangeLogPO controllerChangeLogPO) {
|
||||
controllerChangeLogDAO.insert(controllerChangeLogPO);
|
||||
return controllerChangeLogPO.getId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long queryCount(Long clusterPhyId, PaginationSortDTO dto) {
|
||||
return controllerChangeLogDAO.queryCount(getPaginationQueryParams(clusterPhyId, dto));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ControllerChangeLogPO> paginationQuery(Long clusterPhyId, PaginationSortDTO dto) {
|
||||
return controllerChangeLogDAO.paginationQuery(getPaginationQueryParams(clusterPhyId, dto));
|
||||
}
|
||||
|
||||
private Map<String, Object> getPaginationQueryParams(Long clusterPhyId, PaginationSortDTO dto) {
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("clusterPhyId", clusterPhyId);
|
||||
params.put("sortField", dto.getSortField());
|
||||
params.put("sortType", dto.getSortType());
|
||||
params.put("pageNo", dto.getPageNo());
|
||||
params.put("pageSize", dto.getPageSize());
|
||||
params.put("searchKeywords", dto.getSearchKeywords());
|
||||
return params;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
||||
/**
|
||||
* @author zengqiao
|
||||
* @date 22/6/14
|
||||
*/
|
||||
@Getter
|
||||
@Service
|
||||
public class ConfigUtils {
|
||||
private ConfigUtils() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.config;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.config.PlatformClusterConfigPO;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 平台集群配置
|
||||
* 与Logi-Common中的配置的差别在于该Service的配置是集群相关的配置
|
||||
*/
|
||||
public interface PlatformClusterConfigService {
|
||||
Result<Void> batchReplace(List<PlatformClusterConfigPO> poList, String operator);
|
||||
|
||||
List<PlatformClusterConfigPO> getByClusterAndGroup(Long clusterPhyId, String group);
|
||||
|
||||
Map<String, PlatformClusterConfigPO> getByClusterAndGroupWithoutDefault(Long clusterPhyId, String group);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.config.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.config.PlatformClusterConfigPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.config.PlatformClusterConfigService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.config.PlatformClusterConfigDAO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class PlatformClusterConfigServiceImpl implements PlatformClusterConfigService {
|
||||
private static final ILog log = LogFactory.getLog(PlatformClusterConfigServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private PlatformClusterConfigDAO platformClusterConfigDAO;
|
||||
|
||||
@Override
|
||||
public Result<Void> batchReplace(List<PlatformClusterConfigPO> poList, String operator) {
|
||||
try {
|
||||
platformClusterConfigDAO.batchReplace(poList);
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=batchReplace||data={}||errMsg=exception", ConvertUtil.obj2Json(poList), e);
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PlatformClusterConfigPO> getByClusterAndGroup(Long clusterPhyId, String group) {
|
||||
LambdaQueryWrapper<PlatformClusterConfigPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PlatformClusterConfigPO::getClusterId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(PlatformClusterConfigPO::getValueGroup, group);
|
||||
|
||||
return platformClusterConfigDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, PlatformClusterConfigPO> getByClusterAndGroupWithoutDefault(Long clusterPhyId, String group) {
|
||||
LambdaQueryWrapper<PlatformClusterConfigPO> rowLambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
rowLambdaQueryWrapper.eq(PlatformClusterConfigPO::getClusterId, clusterPhyId);
|
||||
rowLambdaQueryWrapper.eq(PlatformClusterConfigPO::getValueGroup, group);
|
||||
|
||||
LambdaQueryWrapper<PlatformClusterConfigPO> defaultLambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
defaultLambdaQueryWrapper.eq(PlatformClusterConfigPO::getClusterId, -1L);
|
||||
defaultLambdaQueryWrapper.eq(PlatformClusterConfigPO::getValueGroup, group);
|
||||
|
||||
List<PlatformClusterConfigPO> rowPOList = platformClusterConfigDAO.selectList(rowLambdaQueryWrapper);
|
||||
List<PlatformClusterConfigPO> defaultPOList = platformClusterConfigDAO.selectList(defaultLambdaQueryWrapper);
|
||||
|
||||
Map<String, PlatformClusterConfigPO> configPOMap = rowPOList.stream().collect(Collectors.toMap(PlatformClusterConfigPO::getValueName, Function.identity()));
|
||||
for (PlatformClusterConfigPO defaultPO: defaultPOList) {
|
||||
// 修改集群ID
|
||||
defaultPO.setClusterId(clusterPhyId);
|
||||
|
||||
// 存储到Map中
|
||||
configPOMap.putIfAbsent(defaultPO.getValueName(), defaultPO);
|
||||
}
|
||||
|
||||
return configPOMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.group;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricGroupPartitionDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface GroupMetricService {
|
||||
/**
|
||||
* 从Kafka查询指标
|
||||
* @param clusterId 集群ID
|
||||
* @param metric
|
||||
* @return
|
||||
*/
|
||||
Result<List<GroupMetrics>> collectGroupMetricsFromKafka(Long clusterId, String groupName, String metric);
|
||||
Result<List<GroupMetrics>> collectGroupMetricsFromKafka(Long clusterId, String groupName, List<String> metrics);
|
||||
|
||||
/**
|
||||
* 从ES查询指标
|
||||
* @param clusterId 集群ID
|
||||
* @param dto 查询dto
|
||||
* @return
|
||||
*/
|
||||
Result<List<MetricMultiLinesVO>> listGroupMetricsFromES(Long clusterId, MetricGroupPartitionDTO dto);
|
||||
|
||||
/**
|
||||
* 获取该 Group&Topic 的最近指标
|
||||
* @param clusterPhyId
|
||||
* @param groupName
|
||||
* @param topicName
|
||||
* @param metricNames
|
||||
* @return
|
||||
*/
|
||||
Result<List<GroupMetrics>> listPartitionLatestMetricsFromES(Long clusterPhyId, String groupName, String topicName, List<String> metricNames);
|
||||
|
||||
/**
|
||||
* 获取 按照Group&Topic维度进行聚合 后的指标
|
||||
* @param clusterPhyId 物理集群ID
|
||||
* @param groupTopicList GroupTopic列表
|
||||
* @param metricNames 指标列表
|
||||
* @param aggType 聚合类型：avg、max、min、sum
|
||||
* @return
|
||||
*/
|
||||
Result<List<GroupMetrics>> listLatestMetricsAggByGroupTopicFromES(Long clusterPhyId, List<GroupTopic> groupTopicList, List<String> metricNames, AggTypeEnum aggType);
|
||||
|
||||
/**
|
||||
* 统计一段时间内，某个指标出现某个值的次数
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param groupName 消费组名称
|
||||
* @param term 指标
|
||||
* @param startTime 起始时间
|
||||
* @param endTime 结束时间
|
||||
* @return
|
||||
*/
|
||||
Result<Integer> countMetricValueOccurrencesFromES(Long clusterPhyId, String groupName, SearchTerm term, Long startTime, Long endTime);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.group;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import org.apache.kafka.clients.admin.ConsumerGroupDescription;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface GroupService {
|
||||
/**
|
||||
* 从Kafka中获取消费组
|
||||
* @param clusterPhyId 集群ID
|
||||
* @return
|
||||
* @throws NotExistException
|
||||
* @throws AdminOperateException
|
||||
*/
|
||||
List<String> listGroupsFromKafka(Long clusterPhyId) throws NotExistException, AdminOperateException;
|
||||
|
||||
Map<TopicPartition, Long> getGroupOffset(Long clusterPhyId, String groupName) throws NotExistException, AdminOperateException;
|
||||
|
||||
ConsumerGroupDescription getGroupDescription(Long clusterPhyId, String groupName) throws NotExistException, AdminOperateException;
|
||||
|
||||
int replaceDBData(GroupMemberPO groupMemberPO);
|
||||
|
||||
GroupStateEnum getGroupStateFromDB(Long clusterPhyId, String groupName);
|
||||
|
||||
List<GroupMemberPO> listGroupByTopic(Long clusterPhyId, String topicName);
|
||||
|
||||
PaginationResult<GroupMemberPO> pagingGroupMembers(Long clusterPhyId,
|
||||
String topicName,
|
||||
String groupName,
|
||||
String searchTopicKeyword,
|
||||
String searchGroupKeyword,
|
||||
PaginationBaseDTO dto);
|
||||
|
||||
int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime);
|
||||
|
||||
List<String> getGroupsFromDB(Long clusterPhyId);
|
||||
|
||||
GroupMemberPO getGroupFromDB(Long clusterPhyId, String groupName, String topicName);
|
||||
|
||||
Integer calGroupCount(Long clusterPhyId);
|
||||
|
||||
Integer calGroupStatCount(Long clusterPhyId, GroupStateEnum stateEnum);
|
||||
|
||||
Result<Void> resetGroupOffsets(Long clusterPhyId, String groupName, Map<TopicPartition, Long> offsetMap, String operator) throws NotExistException, AdminOperateException;
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.group.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.google.common.collect.Table;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricGroupPartitionDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.group.GroupTopic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.GroupMetricParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.GroupMetricPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BeanUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.score.HealthScoreService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.GroupMetricESDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import org.apache.kafka.clients.admin.OffsetSpec;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.GroupMetricVersionItems.*;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service("groupMetricService")
|
||||
public class GroupMetricServiceImpl extends BaseMetricService implements GroupMetricService {
|
||||
private static final ILog LOGGER = LogFactory.getLog( GroupMetricServiceImpl.class);
|
||||
|
||||
public static final String GROUP_METHOD_GET_JUST_FRO_TEST = "getMetricJustForTest";
|
||||
public static final String GROUP_METHOD_GET_HEALTH_SCORE = "getMetricHealthScore";
|
||||
public static final String GROUP_METHOD_GET_LAG_RELEVANT_FROM_ADMIN_CLIENT = "getLagRelevantFromAdminClient";
|
||||
public static final String GROUP_METHOD_GET_STATE = "getGroupState";
|
||||
|
||||
@Override
|
||||
protected List<String> listMetricPOFields(){
|
||||
return BeanUtil.listBeanFields(GroupMetricPO.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initRegisterVCHandler(){
|
||||
registerVCHandler( GROUP_METHOD_GET_JUST_FRO_TEST, this::getMetricJustForTest);
|
||||
registerVCHandler( GROUP_METHOD_GET_LAG_RELEVANT_FROM_ADMIN_CLIENT, this::getLagRelevantFromAdminClient );
|
||||
registerVCHandler( GROUP_METHOD_GET_HEALTH_SCORE, this::getMetricHealthScore);
|
||||
registerVCHandler( GROUP_METHOD_GET_STATE, this::getGroupState);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private GroupService groupService;
|
||||
|
||||
@Autowired
|
||||
private HealthScoreService healthScoreService;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private GroupMetricESDAO groupMetricESDAO;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.METRIC_GROUP;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<GroupMetrics>> collectGroupMetricsFromKafka(Long clusterId, String groupName, String metric) {
|
||||
try {
|
||||
GroupMetricParam groupMetricParam = new GroupMetricParam(clusterId, groupName, metric);
|
||||
return (Result<List<GroupMetrics>>) doVCHandler(clusterId, metric, groupMetricParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<GroupMetrics>> collectGroupMetricsFromKafka(Long clusterId, String groupName, List<String> metrics) {
|
||||
List<GroupMetrics> allGroupMetrics = new ArrayList<>();
|
||||
Map<String, GroupMetrics> topicPartitionGroupMap = new HashMap<>();
|
||||
|
||||
GroupMetrics groupMetrics = new GroupMetrics(clusterId, groupName, true);
|
||||
for(String metric : metrics){
|
||||
if(null != groupMetrics.getMetrics().get(metric)){continue;}
|
||||
|
||||
Result<List<GroupMetrics>> ret = collectGroupMetricsFromKafka(clusterId, groupName, metric);
|
||||
if(null != ret && ret.successful()){
|
||||
List<GroupMetrics> groupMetricsList = ret.getData();
|
||||
for(GroupMetrics gm : groupMetricsList){
|
||||
if(gm.isBGroupMetric()){
|
||||
groupMetrics.getMetrics().putAll(gm.getMetrics());
|
||||
}else {
|
||||
GroupMetrics topicGroupMetric = topicPartitionGroupMap.getOrDefault(
|
||||
gm.getTopic() + gm.getPartitionId(),
|
||||
new GroupMetrics(clusterId, groupName, false));
|
||||
|
||||
topicGroupMetric.getMetrics().putAll(gm.getMetrics());
|
||||
topicPartitionGroupMap.put(gm.getTopic() + gm.getPartitionId(), topicGroupMetric);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
allGroupMetrics.add(groupMetrics);
|
||||
allGroupMetrics.addAll(topicPartitionGroupMap.values());
|
||||
return Result.buildSuc(allGroupMetrics);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricMultiLinesVO>> listGroupMetricsFromES(Long clusterId, MetricGroupPartitionDTO dto) {
|
||||
Table<String/*metric*/, String/*topic&partition*/, List<MetricPointVO>> retTable = groupMetricESDAO.listGroupMetrics(
|
||||
clusterId, dto.getGroup(), dto.getGroupTopics(), dto.getMetricsNames(),
|
||||
dto.getAggType(), dto.getStartTime(), dto.getEndTime());
|
||||
|
||||
List<MetricMultiLinesVO> multiLinesVOS = metricMap2VO(clusterId, retTable.rowMap());
|
||||
return Result.buildSuc(multiLinesVOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<GroupMetrics>> listLatestMetricsAggByGroupTopicFromES(Long clusterPhyId, List<GroupTopic> groupTopicList,
|
||||
List<String> metricNames, AggTypeEnum aggType) {
|
||||
List<GroupMetricPO> groupMetricPOS = groupMetricESDAO.listLatestMetricsAggByGroupTopic(
|
||||
clusterPhyId, groupTopicList, metricNames, aggType);
|
||||
|
||||
return Result.buildSuc( ConvertUtil.list2List(groupMetricPOS, GroupMetrics.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<GroupMetrics>> listPartitionLatestMetricsFromES(Long clusterPhyId, String groupName, String topicName,
|
||||
List<String> metricNames) {
|
||||
List<GroupMetricPO> groupMetricPOS = groupMetricESDAO.listPartitionLatestMetrics(
|
||||
clusterPhyId, groupName, topicName, metricNames);
|
||||
|
||||
return Result.buildSuc( ConvertUtil.list2List(groupMetricPOS, GroupMetrics.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Integer> countMetricValueOccurrencesFromES(Long clusterPhyId, String groupName,
|
||||
SearchTerm term, Long startTime, Long endTime) {
|
||||
setQueryMetricFlag(term);
|
||||
int count = groupMetricESDAO.countMetricValue(clusterPhyId, groupName,
|
||||
term, startTime, endTime);
|
||||
|
||||
if(count < 0){
|
||||
return Result.buildFail();
|
||||
}
|
||||
|
||||
return Result.buildSuc(count);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
/**
|
||||
* 获取Lag相关指标, 包括lag，consumedOffset partitionEndOffset
|
||||
*/
|
||||
private Result<List<GroupMetrics>> getLagRelevantFromAdminClient(VersionItemParam param) {
|
||||
GroupMetricParam groupMetricParam = (GroupMetricParam)param;
|
||||
Long clusterId = groupMetricParam.getClusterPhyId();
|
||||
String groupName = groupMetricParam.getGroupName();
|
||||
String metric = groupMetricParam.getMetric();
|
||||
|
||||
List<GroupMetrics> metricsList = new ArrayList<>();
|
||||
try {
|
||||
Map<TopicPartition, Long> groupOffsetMap = groupService.getGroupOffset(clusterId, groupName);
|
||||
|
||||
// 组织 GROUP_METRIC_OFFSET_CONSUMED 指标
|
||||
for (Map.Entry<TopicPartition, Long> entry: groupOffsetMap.entrySet()) {
|
||||
GroupMetrics metrics = new GroupMetrics(clusterId, entry.getKey().partition(), entry.getKey().topic(), groupName, false);
|
||||
metrics.putMetric(GROUP_METRIC_OFFSET_CONSUMED, entry.getValue().floatValue());
|
||||
|
||||
metricsList.add(metrics);
|
||||
}
|
||||
|
||||
for (String topicName: groupOffsetMap.keySet().stream().map(elem -> elem.topic()).collect(Collectors.toSet())) {
|
||||
Result<Map<TopicPartition, Long>> offsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterId, topicName, OffsetSpec.latest(), null);
|
||||
if (!offsetMapResult.hasData()) {
|
||||
// 这个分区获取失败
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Map.Entry<TopicPartition, Long> entry: offsetMapResult.getData().entrySet()) {
|
||||
// 组织 GROUP_METRIC_LOG_END_OFFSET 指标
|
||||
GroupMetrics metrics = new GroupMetrics(clusterId, entry.getKey().partition(), entry.getKey().topic(), groupName, false);
|
||||
metrics.putMetric(GROUP_METRIC_LOG_END_OFFSET, entry.getValue().floatValue());
|
||||
metricsList.add(metrics);
|
||||
|
||||
Long groupOffset = groupOffsetMap.get(entry.getKey());
|
||||
if (groupOffset == null) {
|
||||
// 不存在，则直接跳过
|
||||
continue;
|
||||
}
|
||||
|
||||
// 组织 GROUP_METRIC_LAG 指标
|
||||
GroupMetrics groupMetrics = new GroupMetrics(clusterId, entry.getKey().partition(), entry.getKey().topic(), groupName, false);
|
||||
groupMetrics.putMetric(GROUP_METRIC_LAG, Math.max(0L, entry.getValue() - groupOffset) * 1.0f);
|
||||
|
||||
metricsList.add(groupMetrics);
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(metricsList);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("class=GroupMetricServiceImpl||method=getLagFromAdminClient||clusterPhyId={}||groupName={}||metrics={}||msg=exception", clusterId, groupName, metric, e);
|
||||
return Result.buildFailure(VC_KAFKA_CLIENT_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<GroupMetrics>> getMetricJustForTest(VersionItemParam param) {
|
||||
GroupMetricParam groupMetricParam = (GroupMetricParam)param;
|
||||
Long clusterPhyId = groupMetricParam.getClusterPhyId();
|
||||
String groupName = groupMetricParam.getGroupName();
|
||||
String metric = groupMetricParam.getMetric();
|
||||
|
||||
GroupMetrics groupMetrics = new GroupMetrics(clusterPhyId, groupName, true);
|
||||
groupMetrics.putMetric(metric, 123f);
|
||||
|
||||
return Result.buildSuc( Arrays.asList(groupMetrics));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取group的状态
|
||||
* @param param
|
||||
* @return
|
||||
*/
|
||||
private Result<List<GroupMetrics>> getGroupState(VersionItemParam param) {
|
||||
GroupMetricParam groupMetricParam = (GroupMetricParam)param;
|
||||
Long clusterPhyId = groupMetricParam.getClusterPhyId();
|
||||
String groupName = groupMetricParam.getGroupName();
|
||||
String metric = groupMetricParam.getMetric();
|
||||
|
||||
GroupMetrics metrics = new GroupMetrics(clusterPhyId, groupName, true);
|
||||
metrics.putMetric(metric,
|
||||
(float)groupService.getGroupStateFromDB(clusterPhyId, groupName).getCode()
|
||||
);
|
||||
|
||||
return Result.buildSuc(Arrays.asList(metrics));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取group的健康分指标
|
||||
* @param param
|
||||
* @return
|
||||
*/
|
||||
private Result<List<GroupMetrics>> getMetricHealthScore(VersionItemParam param) {
|
||||
GroupMetricParam groupMetricParam = (GroupMetricParam)param;
|
||||
|
||||
return Result.buildSuc(Arrays.asList(healthScoreService.calGroupHealthScore(
|
||||
groupMetricParam.getClusterPhyId(), groupMetricParam.getGroupName()))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.group.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.group.GroupMemberPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.group.GroupMemberDAO;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.SERVICE_SEARCH_GROUP;
|
||||
|
||||
@Service
|
||||
public class GroupServiceImpl extends BaseVersionControlService implements GroupService {
|
||||
private static final ILog log = LogFactory.getLog(GroupServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private GroupMemberDAO groupMemberDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return SERVICE_SEARCH_GROUP;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listGroupsFromKafka(Long clusterPhyId) throws NotExistException, AdminOperateException {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhyId);
|
||||
|
||||
try {
|
||||
ListConsumerGroupsResult listConsumerGroupsResult = adminClient.listConsumerGroups(
|
||||
new ListConsumerGroupsOptions()
|
||||
.timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
List<String> groupNameList = new ArrayList<>();
|
||||
for (ConsumerGroupListing consumerGroupListing: listConsumerGroupsResult.all().get()) {
|
||||
groupNameList.add(consumerGroupListing.groupId());
|
||||
}
|
||||
|
||||
return groupNameList;
|
||||
} catch (Exception e) {
|
||||
log.error("method=getGroupsFromKafka||clusterPhyId={}||errMsg=exception!", clusterPhyId, e);
|
||||
|
||||
throw new AdminOperateException(e.getMessage(), e, ResultStatus.KAFKA_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<TopicPartition, Long> getGroupOffset(Long clusterPhyId, String groupName) throws NotExistException, AdminOperateException {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhyId);
|
||||
|
||||
Map<TopicPartition, Long> offsetMap = new HashMap<>();
|
||||
try {
|
||||
ListConsumerGroupOffsetsResult listConsumerGroupOffsetsResult = adminClient.listConsumerGroupOffsets(groupName);
|
||||
Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap = listConsumerGroupOffsetsResult.partitionsToOffsetAndMetadata().get();
|
||||
for (Map.Entry<TopicPartition, OffsetAndMetadata> entry: offsetAndMetadataMap.entrySet()) {
|
||||
offsetMap.put(entry.getKey(), entry.getValue().offset());
|
||||
}
|
||||
|
||||
return offsetMap;
|
||||
} catch (Exception e) {
|
||||
log.error("method=getGroupOffset||clusterPhyId={}|groupName={}||errMsg=exception!", clusterPhyId, groupName, e);
|
||||
|
||||
throw new AdminOperateException(e.getMessage(), e, ResultStatus.KAFKA_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConsumerGroupDescription getGroupDescription(Long clusterPhyId, String groupName) throws NotExistException, AdminOperateException {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhyId);
|
||||
|
||||
try {
|
||||
DescribeConsumerGroupsResult describeConsumerGroupsResult = adminClient.describeConsumerGroups(Arrays.asList(groupName), new DescribeConsumerGroupsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS).includeAuthorizedOperations(true));
|
||||
|
||||
return describeConsumerGroupsResult.all().get().get(groupName);
|
||||
} catch(Exception e){
|
||||
log.error("method=getGroupDescription||clusterPhyId={}|groupName={}||errMsg=exception!", clusterPhyId, groupName, e);
|
||||
|
||||
throw new AdminOperateException(e.getMessage(), e, ResultStatus.KAFKA_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int replaceDBData(GroupMemberPO groupMemberPO) {
|
||||
return groupMemberDAO.replace(groupMemberPO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GroupStateEnum getGroupStateFromDB(Long clusterPhyId, String groupName) {
|
||||
LambdaQueryWrapper<GroupMemberPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(GroupMemberPO::getGroupName, groupName);
|
||||
|
||||
List<GroupMemberPO> poList = groupMemberDAO.selectList(lambdaQueryWrapper);
|
||||
if (poList == null || poList.isEmpty()) {
|
||||
return GroupStateEnum.UNKNOWN;
|
||||
}
|
||||
|
||||
return GroupStateEnum.getByState(poList.get(0).getState());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<GroupMemberPO> listGroupByTopic(Long clusterPhyId, String topicName) {
|
||||
LambdaQueryWrapper<GroupMemberPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(GroupMemberPO::getTopicName, topicName);
|
||||
|
||||
return groupMemberDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<GroupMemberPO> pagingGroupMembers(Long clusterPhyId,
|
||||
String topicName,
|
||||
String groupName,
|
||||
String searchTopicKeyword,
|
||||
String searchGroupKeyword,
|
||||
PaginationBaseDTO dto) {
|
||||
LambdaQueryWrapper<GroupMemberPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(!ValidateUtils.isBlank(topicName), GroupMemberPO::getTopicName, topicName);
|
||||
lambdaQueryWrapper.eq(!ValidateUtils.isBlank(groupName), GroupMemberPO::getGroupName, groupName);
|
||||
lambdaQueryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.like(!ValidateUtils.isBlank(searchTopicKeyword), GroupMemberPO::getTopicName, searchTopicKeyword);
|
||||
lambdaQueryWrapper.like(!ValidateUtils.isBlank(searchGroupKeyword), GroupMemberPO::getGroupName, searchGroupKeyword);
|
||||
|
||||
IPage<GroupMemberPO> iPage = new Page<>();
|
||||
iPage.setPages(dto.getPageNo());
|
||||
iPage.setSize(dto.getPageSize());
|
||||
|
||||
iPage = groupMemberDAO.selectPage(iPage, lambdaQueryWrapper);
|
||||
|
||||
return PaginationResult.buildSuc(iPage.getRecords(), iPage);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime) {
|
||||
LambdaQueryWrapper<GroupMemberPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.le(GroupMemberPO::getUpdateTime, beforeTime);
|
||||
return groupMemberDAO.delete(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getGroupsFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<GroupMemberPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
List<GroupMemberPO> poList = groupMemberDAO.selectList(queryWrapper);
|
||||
if (poList == null) {
|
||||
poList = new ArrayList<>();
|
||||
}
|
||||
return new ArrayList<>(poList.stream().map(elem -> elem.getGroupName()).collect(Collectors.toSet()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public GroupMemberPO getGroupFromDB(Long clusterPhyId, String groupName, String topicName) {
|
||||
LambdaQueryWrapper<GroupMemberPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.eq(GroupMemberPO::getTopicName, topicName);
|
||||
queryWrapper.eq(GroupMemberPO::getGroupName, groupName);
|
||||
|
||||
return groupMemberDAO.selectOne(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer calGroupCount(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<GroupMemberPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
List<GroupMemberPO> poList = groupMemberDAO.selectList(queryWrapper);
|
||||
if (poList == null) {
|
||||
poList = new ArrayList<>();
|
||||
}
|
||||
|
||||
return poList.stream().map(elem -> elem.getGroupName()).collect(Collectors.toSet()).size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer calGroupStatCount(Long clusterPhyId, GroupStateEnum stateEnum) {
|
||||
LambdaQueryWrapper<GroupMemberPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(GroupMemberPO::getClusterPhyId, clusterPhyId);
|
||||
queryWrapper.eq(GroupMemberPO::getState, stateEnum.getState());
|
||||
|
||||
List<GroupMemberPO> poList = groupMemberDAO.selectList(queryWrapper);
|
||||
if (poList == null) {
|
||||
poList = new ArrayList<>();
|
||||
}
|
||||
|
||||
return poList.stream().map(elem -> elem.getGroupName()).collect(Collectors.toSet()).size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> resetGroupOffsets(Long clusterPhyId,
|
||||
String groupName,
|
||||
Map<TopicPartition, Long> resetOffsetMap,
|
||||
String operator) throws NotExistException, AdminOperateException {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhyId);
|
||||
|
||||
try {
|
||||
Map<TopicPartition, OffsetAndMetadata> offsets = resetOffsetMap.entrySet().stream().collect(Collectors.toMap(
|
||||
elem -> elem.getKey(),
|
||||
elem -> new OffsetAndMetadata(elem.getValue()),
|
||||
(key1 , key2) -> key2
|
||||
));
|
||||
|
||||
AlterConsumerGroupOffsetsResult alterConsumerGroupOffsetsResult = adminClient.alterConsumerGroupOffsets(
|
||||
groupName,
|
||||
offsets,
|
||||
new AlterConsumerGroupOffsetsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
alterConsumerGroupOffsetsResult.all().get();
|
||||
OplogDTO oplogDTO = new OplogDTO(operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.KAFKA_GROUP.getDesc(),
|
||||
String.format("clusterPhyId:%d groupName:%s", clusterPhyId, groupName),
|
||||
ConvertUtil.obj2Json(resetOffsetMap));
|
||||
opLogWrapService.saveOplogAndIgnoreException(oplogDTO);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch(Exception e){
|
||||
log.error("method=resetGroupOffsets||clusterPhyId={}|groupName={}||errMsg=exception!", clusterPhyId, groupName, e);
|
||||
|
||||
throw new AdminOperateException(e.getMessage(), e, ResultStatus.KAFKA_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checker;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.Function;
|
||||
|
||||
@Data
|
||||
public abstract class AbstractHealthCheckService {
|
||||
private static final ILog log = LogFactory.getLog(AbstractHealthCheckService.class);
|
||||
|
||||
protected static final Map<
|
||||
String,
|
||||
Function<Tuple<ClusterPhyParam, BaseClusterHealthConfig>, HealthCheckResult>
|
||||
> functionMap = new ConcurrentHashMap<>();
|
||||
|
||||
public abstract List<ClusterPhyParam> getResList(Long clusterPhyId);
|
||||
|
||||
public abstract HealthCheckDimensionEnum getHealthCheckDimensionEnum();
|
||||
|
||||
public HealthCheckResult checkAndGetResult(ClusterPhyParam clusterPhyParam, BaseClusterHealthConfig clusterHealthConfig) {
|
||||
if (ValidateUtils.anyNull(clusterPhyParam.getClusterPhyId(), clusterPhyParam, clusterHealthConfig)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
HealthCheckDimensionEnum dimensionEnum = this.getHealthCheckDimensionEnum();
|
||||
if (!clusterHealthConfig.getCheckNameEnum().getDimensionEnum().equals(dimensionEnum)) {
|
||||
// 类型不匹配
|
||||
return null;
|
||||
}
|
||||
|
||||
Function<Tuple<ClusterPhyParam, BaseClusterHealthConfig>, HealthCheckResult> function = functionMap.get(clusterHealthConfig.getCheckNameEnum().getConfigName());
|
||||
if (function == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return function.apply(new Tuple<>(clusterPhyParam, clusterHealthConfig));
|
||||
} catch (Exception e) {
|
||||
log.error("method=checkAndGetResult||clusterPhyParam={}||clusterHealthConfig={}||errMsg=exception!",
|
||||
clusterPhyParam, clusterHealthConfig, e);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checker.broker;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.HealthCompareValueConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.broker.BrokerParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckNameEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.checker.AbstractHealthCheckService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.BrokerMetricVersionItems;
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Service
|
||||
public class HealthCheckBrokerService extends AbstractHealthCheckService {
|
||||
private static final ILog log = LogFactory.getLog(HealthCheckBrokerService.class);
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private BrokerMetricService brokerMetricService;
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
functionMap.putIfAbsent(HealthCheckNameEnum.BROKER_REQUEST_QUEUE_FULL.getConfigName(), this::checkBrokerRequestQueueFull);
|
||||
functionMap.putIfAbsent(HealthCheckNameEnum.BROKER_NETWORK_PROCESSOR_AVG_IDLE_TOO_LOW.getConfigName(), this::checkBrokerNetworkProcessorAvgIdleTooLow);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ClusterPhyParam> getResList(Long clusterPhyId) {
|
||||
List<ClusterPhyParam> paramList = new ArrayList<>();
|
||||
for (Broker broker: brokerService.listAliveBrokersFromDB(clusterPhyId)) {
|
||||
paramList.add(new BrokerParam(clusterPhyId, broker.getBrokerId()));
|
||||
}
|
||||
return paramList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HealthCheckDimensionEnum getHealthCheckDimensionEnum() {
|
||||
return HealthCheckDimensionEnum.BROKER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Broker网络处理线程平均值过低
|
||||
*/
|
||||
private HealthCheckResult checkBrokerNetworkProcessorAvgIdleTooLow(Tuple<ClusterPhyParam, BaseClusterHealthConfig> paramTuple) {
|
||||
BrokerParam param = (BrokerParam) paramTuple.getV1();
|
||||
HealthCompareValueConfig singleConfig = (HealthCompareValueConfig) paramTuple.getV2();
|
||||
|
||||
HealthCheckResult checkResult = new HealthCheckResult(
|
||||
HealthCheckDimensionEnum.BROKER.getDimension(),
|
||||
HealthCheckNameEnum.BROKER_NETWORK_PROCESSOR_AVG_IDLE_TOO_LOW.getConfigName(),
|
||||
param.getClusterPhyId(),
|
||||
String.valueOf(param.getBrokerId())
|
||||
);
|
||||
|
||||
Result<BrokerMetrics> metricsResult = brokerMetricService.getLatestMetricsFromES(
|
||||
param.getClusterPhyId(), param.getBrokerId());
|
||||
|
||||
if (metricsResult.failed()) {
|
||||
log.error("method=checkBrokerNetworkProcessorAvgIdleTooLow||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, singleConfig, metricsResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
Float avgIdle = metricsResult.getData().getMetrics().get( BrokerMetricVersionItems.BROKER_METRIC_NETWORK_RPO_AVG_IDLE);
|
||||
if (avgIdle == null) {
|
||||
log.error("method=checkBrokerNetworkProcessorAvgIdleTooLow||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, singleConfig, metricsResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
checkResult.setPassed(avgIdle >= singleConfig.getValue()? 1: 0);
|
||||
|
||||
return checkResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Broker请求队列满
|
||||
*/
|
||||
private HealthCheckResult checkBrokerRequestQueueFull(Tuple<ClusterPhyParam, BaseClusterHealthConfig> paramTuple) {
|
||||
BrokerParam param = (BrokerParam) paramTuple.getV1();
|
||||
HealthCompareValueConfig singleConfig = (HealthCompareValueConfig) paramTuple.getV2();
|
||||
|
||||
HealthCheckResult checkResult = new HealthCheckResult(
|
||||
HealthCheckDimensionEnum.BROKER.getDimension(),
|
||||
HealthCheckNameEnum.BROKER_REQUEST_QUEUE_FULL.getConfigName(),
|
||||
param.getClusterPhyId(),
|
||||
String.valueOf(param.getBrokerId())
|
||||
);
|
||||
|
||||
Result<BrokerMetrics> metricsResult = brokerMetricService.collectBrokerMetricsFromKafka(
|
||||
param.getClusterPhyId(),
|
||||
param.getBrokerId(),
|
||||
Arrays.asList( BrokerMetricVersionItems.BROKER_METRIC_TOTAL_REQ_QUEUE)
|
||||
);
|
||||
|
||||
if (metricsResult.failed()) {
|
||||
log.error("method=checkBrokerRequestQueueFull||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, singleConfig, metricsResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
Float queueSize = metricsResult.getData().getMetrics().get( BrokerMetricVersionItems.BROKER_METRIC_TOTAL_REQ_QUEUE);
|
||||
if (queueSize == null) {
|
||||
log.error("method=checkBrokerRequestQueueFull||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, singleConfig, metricsResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
checkResult.setPassed(queueSize <= singleConfig.getValue()? Constant.YES : Constant.NO);
|
||||
|
||||
return checkResult;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checker.cluster;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.HealthCompareValueConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckNameEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.checker.AbstractHealthCheckService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.ClusterMetricVersionItems;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
public class HealthCheckClusterService extends AbstractHealthCheckService {
|
||||
private static final ILog log = LogFactory.getLog(HealthCheckClusterService.class);
|
||||
|
||||
@Autowired
|
||||
private ClusterMetricService clusterMetricService;
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
functionMap.putIfAbsent(HealthCheckNameEnum.CLUSTER_NO_CONTROLLER.getConfigName(), this::checkClusterNoController);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ClusterPhyParam> getResList(Long clusterPhyId) {
|
||||
return Arrays.asList(new ClusterPhyParam(clusterPhyId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public HealthCheckDimensionEnum getHealthCheckDimensionEnum() {
|
||||
return HealthCheckDimensionEnum.CLUSTER;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查NoController
|
||||
*/
|
||||
private HealthCheckResult checkClusterNoController(Tuple<ClusterPhyParam, BaseClusterHealthConfig> singleConfigSimpleTuple) {
|
||||
ClusterPhyParam param = singleConfigSimpleTuple.getV1();
|
||||
HealthCompareValueConfig valueConfig = (HealthCompareValueConfig) singleConfigSimpleTuple.getV2();
|
||||
|
||||
Result<ClusterMetrics> clusterMetricsResult = clusterMetricService.getLatestMetricsFromES(param.getClusterPhyId(), Arrays.asList(ClusterMetricVersionItems.CLUSTER_METRIC_ACTIVE_CONTROLLER_COUNT));
|
||||
if (clusterMetricsResult.failed() || !clusterMetricsResult.hasData()) {
|
||||
log.error("method=checkClusterNoController||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, valueConfig, clusterMetricsResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
HealthCheckResult checkResult = new HealthCheckResult(
|
||||
HealthCheckDimensionEnum.CLUSTER.getDimension(),
|
||||
HealthCheckNameEnum.CLUSTER_NO_CONTROLLER.getConfigName(),
|
||||
param.getClusterPhyId(),
|
||||
""
|
||||
);
|
||||
|
||||
Float activeController = clusterMetricsResult.getData().getMetric(ClusterMetricVersionItems.CLUSTER_METRIC_ACTIVE_CONTROLLER_COUNT);
|
||||
|
||||
|
||||
checkResult.setPassed(activeController.intValue() != valueConfig.getValue().intValue() ? 0: 1);
|
||||
|
||||
return checkResult;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checker.group;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.HealthDetectedInLatestMinutesConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.group.GroupParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.group.GroupStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckNameEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.group.GroupService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.checker.AbstractHealthCheckService;
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.GroupMetricVersionItems.GROUP_METRIC_STATE;
|
||||
|
||||
@Data
|
||||
@Service
|
||||
public class HealthCheckGroupService extends AbstractHealthCheckService {
|
||||
private static final ILog log = LogFactory.getLog(HealthCheckGroupService.class);
|
||||
|
||||
@Autowired
|
||||
private GroupService groupService;
|
||||
|
||||
@Autowired
|
||||
private GroupMetricService groupMetricService;
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
functionMap.putIfAbsent(HealthCheckNameEnum.GROUP_RE_BALANCE_TOO_FREQUENTLY.getConfigName(), this::checkReBalanceTooFrequently);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ClusterPhyParam> getResList(Long clusterPhyId) {
|
||||
return groupService.getGroupsFromDB(clusterPhyId).stream().map(elem -> new GroupParam(clusterPhyId, elem)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public HealthCheckDimensionEnum getHealthCheckDimensionEnum() {
|
||||
return HealthCheckDimensionEnum.GROUP;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查Group re-balance太频繁
|
||||
*/
|
||||
private HealthCheckResult checkReBalanceTooFrequently(Tuple<ClusterPhyParam, BaseClusterHealthConfig> paramTuple) {
|
||||
GroupParam param = (GroupParam) paramTuple.getV1();
|
||||
HealthDetectedInLatestMinutesConfig singleConfig = (HealthDetectedInLatestMinutesConfig) paramTuple.getV2();
|
||||
|
||||
HealthCheckResult checkResult = new HealthCheckResult(
|
||||
HealthCheckDimensionEnum.GROUP.getDimension(),
|
||||
HealthCheckNameEnum.GROUP_RE_BALANCE_TOO_FREQUENTLY.getConfigName(),
|
||||
param.getClusterPhyId(),
|
||||
param.getGroupName()
|
||||
);
|
||||
|
||||
Result<Integer> countResult = groupMetricService.countMetricValueOccurrencesFromES(
|
||||
param.getClusterPhyId(),
|
||||
param.getGroupName(),
|
||||
new SearchTerm(GROUP_METRIC_STATE, GroupStateEnum.PREPARING_RE_BALANCE.getCode().toString(), true),
|
||||
System.currentTimeMillis() - singleConfig.getLatestMinutes() * 60L * 1000L,
|
||||
System.currentTimeMillis()
|
||||
);
|
||||
|
||||
if (countResult.failed() || !countResult.hasData()) {
|
||||
log.error("method=checkReBalanceTooFrequently||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, singleConfig, countResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
checkResult.setPassed(countResult.getData() >= singleConfig.getDetectedTimes()? 0: 1);
|
||||
|
||||
return checkResult;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checker.topic;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.HealthCompareValueConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.HealthDetectedInLatestMinutesConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.cluster.ClusterPhyParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckNameEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.checker.AbstractHealthCheckService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.TopicMetricVersionItems.TOPIC_METRIC_UNDER_REPLICA_PARTITIONS;
|
||||
|
||||
@Service
|
||||
public class HealthCheckTopicService extends AbstractHealthCheckService {
|
||||
private static final ILog log = LogFactory.getLog(HealthCheckTopicService.class);
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private TopicMetricService topicMetricService;
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
functionMap.putIfAbsent(HealthCheckNameEnum.TOPIC_NO_LEADER.getConfigName(), this::checkTopicNoLeader);
|
||||
functionMap.putIfAbsent(HealthCheckNameEnum.TOPIC_UNDER_REPLICA_TOO_LONG.getConfigName(), this::checkTopicUnderReplicatedPartition);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ClusterPhyParam> getResList(Long clusterPhyId) {
|
||||
List<ClusterPhyParam> paramList = new ArrayList<>();
|
||||
for (Topic topic: topicService.listTopicsFromDB(clusterPhyId)) {
|
||||
paramList.add(new TopicParam(clusterPhyId, topic.getTopicName()));
|
||||
}
|
||||
return paramList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HealthCheckDimensionEnum getHealthCheckDimensionEnum() {
|
||||
return HealthCheckDimensionEnum.TOPIC;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查Topic长期未同步
|
||||
*/
|
||||
private HealthCheckResult checkTopicUnderReplicatedPartition(Tuple<ClusterPhyParam, BaseClusterHealthConfig> paramTuple) {
|
||||
TopicParam param = (TopicParam) paramTuple.getV1();
|
||||
HealthDetectedInLatestMinutesConfig singleConfig = (HealthDetectedInLatestMinutesConfig) paramTuple.getV2();
|
||||
|
||||
HealthCheckResult checkResult = new HealthCheckResult(
|
||||
HealthCheckDimensionEnum.TOPIC.getDimension(),
|
||||
HealthCheckNameEnum.TOPIC_UNDER_REPLICA_TOO_LONG.getConfigName(),
|
||||
param.getClusterPhyId(),
|
||||
param.getTopicName()
|
||||
);
|
||||
|
||||
Result<Integer> countResult = topicMetricService.countMetricValueOccurrencesFromES(
|
||||
param.getClusterPhyId(),
|
||||
param.getTopicName(),
|
||||
new SearchTerm(TOPIC_METRIC_UNDER_REPLICA_PARTITIONS, "0", false),
|
||||
System.currentTimeMillis() - singleConfig.getLatestMinutes() * 60L * 1000L,
|
||||
System.currentTimeMillis()
|
||||
);
|
||||
|
||||
if (countResult.failed() || !countResult.hasData()) {
|
||||
log.error("method=checkTopicUnderReplicatedPartition||param={}||config={}||result={}||errMsg=get metrics failed",
|
||||
param, singleConfig, countResult);
|
||||
return null;
|
||||
}
|
||||
|
||||
checkResult.setPassed(countResult.getData() >= singleConfig.getDetectedTimes()? 0: 1);
|
||||
return checkResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查NoLeader
|
||||
*/
|
||||
private HealthCheckResult checkTopicNoLeader(Tuple<ClusterPhyParam, BaseClusterHealthConfig> singleConfigSimpleTuple) {
|
||||
TopicParam param = (TopicParam) singleConfigSimpleTuple.getV1();
|
||||
List<Partition> partitionList = partitionService.listPartitionFromCacheFirst(param.getClusterPhyId(), param.getTopicName());
|
||||
|
||||
HealthCompareValueConfig valueConfig = (HealthCompareValueConfig) singleConfigSimpleTuple.getV2();
|
||||
HealthCheckResult checkResult = new HealthCheckResult(
|
||||
HealthCheckDimensionEnum.TOPIC.getDimension(),
|
||||
HealthCheckNameEnum.TOPIC_NO_LEADER.getConfigName(),
|
||||
param.getClusterPhyId(),
|
||||
param.getTopicName()
|
||||
);
|
||||
|
||||
checkResult.setPassed(partitionList.stream().filter(elem -> elem.getLeaderBrokerId().equals(Constant.INVALID_CODE)).count() >= valueConfig.getValue()? 0: 1);
|
||||
|
||||
return checkResult;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checkresult;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.health.HealthCheckResultPO;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface HealthCheckResultService {
|
||||
int replace(HealthCheckResult healthCheckResult);
|
||||
|
||||
int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime);
|
||||
|
||||
List<HealthCheckResultPO> getClusterHealthCheckResult(Long clusterPhyId);
|
||||
|
||||
List<HealthCheckResultPO> getClusterResourcesHealthCheckResult(Long clusterPhyId, Integer resDimension);
|
||||
|
||||
List<HealthCheckResultPO> getResHealthCheckResult(Long clusterPhyId, Integer dimension, String resNme);
|
||||
|
||||
Map<String, BaseClusterHealthConfig> getClusterHealthConfig(Long clusterPhyId);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.checkresult.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthCheckResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.config.PlatformClusterConfigPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.health.HealthCheckResultPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.config.ConfigGroupEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckNameEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.config.PlatformClusterConfigService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.checkresult.HealthCheckResultService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.health.HealthCheckResultDAO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
@Service
|
||||
public class HealthCheckResultServiceImpl implements HealthCheckResultService {
|
||||
private static final ILog log = LogFactory.getLog(HealthCheckResultServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private HealthCheckResultDAO healthCheckResultDAO;
|
||||
|
||||
@Autowired
|
||||
private PlatformClusterConfigService platformClusterConfigService;
|
||||
|
||||
@Override
|
||||
public int replace(HealthCheckResult healthCheckResult) {
|
||||
return healthCheckResultDAO.replace(ConvertUtil.obj2Obj(healthCheckResult, HealthCheckResultPO.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteByUpdateTimeBeforeInDB(Long clusterPhyId, Date beforeTime) {
|
||||
LambdaQueryWrapper<HealthCheckResultPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.le(HealthCheckResultPO::getUpdateTime, beforeTime);
|
||||
return healthCheckResultDAO.delete(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HealthCheckResultPO> getClusterHealthCheckResult(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<HealthCheckResultPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getClusterPhyId, clusterPhyId);
|
||||
return healthCheckResultDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HealthCheckResultPO> getClusterResourcesHealthCheckResult(Long clusterPhyId, Integer resDimension) {
|
||||
LambdaQueryWrapper<HealthCheckResultPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getDimension, resDimension);
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getClusterPhyId, clusterPhyId);
|
||||
return healthCheckResultDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HealthCheckResultPO> getResHealthCheckResult(Long clusterPhyId, Integer resDimension, String resNme) {
|
||||
LambdaQueryWrapper<HealthCheckResultPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getDimension, resDimension);
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(HealthCheckResultPO::getResName, resNme);
|
||||
return healthCheckResultDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, BaseClusterHealthConfig> getClusterHealthConfig(Long clusterPhyId) {
|
||||
Map<String, PlatformClusterConfigPO> configPOMap = platformClusterConfigService.getByClusterAndGroupWithoutDefault(clusterPhyId, ConfigGroupEnum.HEALTH.name());
|
||||
|
||||
Map<String, BaseClusterHealthConfig> configMap = new HashMap<>();
|
||||
for (PlatformClusterConfigPO po: configPOMap.values()) {
|
||||
try {
|
||||
HealthCheckNameEnum nameEnum = HealthCheckNameEnum.getByName(po.getValueName());
|
||||
if (HealthCheckNameEnum.UNKNOWN.equals(nameEnum)) {
|
||||
log.warn("method=getClusterHealthConfig||config={}||errMsg=config name illegal", po);
|
||||
continue;
|
||||
}
|
||||
|
||||
BaseClusterHealthConfig healthConfig = (BaseClusterHealthConfig) ConvertUtil.str2ObjByJson(po.getValue(), nameEnum.getConfigClazz());
|
||||
healthConfig.setCheckNameEnum(nameEnum);
|
||||
healthConfig.setClusterPhyId(clusterPhyId);
|
||||
configMap.put(po.getValueName(), healthConfig);
|
||||
} catch (Exception e) {
|
||||
log.error("method=getClusterHealthConfig||config={}||errMsg=exception!", po, e);
|
||||
}
|
||||
}
|
||||
return configMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.score;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthScoreResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface HealthScoreService {
|
||||
/**
|
||||
* 获取集群健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @return
|
||||
*/
|
||||
ClusterMetrics calClusterHealthScore(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取集群Topics健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @return
|
||||
*/
|
||||
ClusterMetrics calClusterTopicsHealthScore(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取集群Brokers健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @return
|
||||
*/
|
||||
ClusterMetrics calClusterBrokersHealthScore(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取集群Groups健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @return
|
||||
*/
|
||||
ClusterMetrics calClusterGroupsHealthScore(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取集群健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param topicName Topic名称
|
||||
* @return
|
||||
*/
|
||||
TopicMetrics calTopicHealthScore(Long clusterPhyId, String topicName);
|
||||
|
||||
/**
|
||||
* 获取集群健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param brokerId brokerId
|
||||
* @return
|
||||
*/
|
||||
BrokerMetrics calBrokerHealthScore(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
/**
|
||||
* 获取集群健康分指标
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param groupName group名称
|
||||
* @return
|
||||
*/
|
||||
GroupMetrics calGroupHealthScore(Long clusterPhyId, String groupName);
|
||||
|
||||
/**
|
||||
* 获取集群健康分结果
|
||||
* @param clusterPhyId 集群ID
|
||||
* @return
|
||||
*/
|
||||
List<HealthScoreResult> getClusterHealthScoreResult(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取集群某个维度健康分结果
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param dimensionEnum 维度
|
||||
* @return
|
||||
*/
|
||||
List<HealthScoreResult> getDimensionHealthScoreResult(Long clusterPhyId, HealthCheckDimensionEnum dimensionEnum);
|
||||
|
||||
/**
|
||||
* 获取集群某个资源的健康分结果
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param dimension 维度
|
||||
* @param resNme 资源名称
|
||||
* @return
|
||||
*/
|
||||
List<HealthScoreResult> getResHealthScoreResult(Long clusterPhyId, Integer dimension, String resNme);
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.health.score.impl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.healthcheck.BaseClusterHealthConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.health.HealthScoreResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ClusterMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.GroupMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.health.HealthCheckResultPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckNameEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.health.HealthCheckDimensionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.checkresult.HealthCheckResultService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.score.HealthScoreService;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.BrokerMetricVersionItems.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.TopicMetricVersionItems.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.ClusterMetricVersionItems.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.GroupMetricVersionItems.*;
|
||||
|
||||
|
||||
@Service
|
||||
public class HealthScoreServiceImpl implements HealthScoreService {
|
||||
@Autowired
|
||||
private HealthCheckResultService healthCheckResultService;
|
||||
|
||||
@Override
|
||||
public ClusterMetrics calClusterHealthScore(Long clusterPhyId) {
|
||||
List<HealthScoreResult> allHealthScoreResultList = this.getClusterHealthScoreResult(clusterPhyId);
|
||||
Map<Integer, List<HealthScoreResult>> healthScoreResultMap = new HashMap<>();
|
||||
for (HealthScoreResult healthScoreResult: allHealthScoreResultList) {
|
||||
healthScoreResultMap.putIfAbsent(healthScoreResult.getCheckNameEnum().getDimensionEnum().getDimension(), new ArrayList<>());
|
||||
healthScoreResultMap.get(healthScoreResult.getCheckNameEnum().getDimensionEnum().getDimension()).add(healthScoreResult);
|
||||
}
|
||||
|
||||
Float healthScore = 0f;
|
||||
Float healthCheckPassed = 0f;
|
||||
Float healthCheckTotal = 0f;
|
||||
|
||||
ClusterMetrics metrics = new ClusterMetrics(clusterPhyId);
|
||||
|
||||
// cluster维度
|
||||
List<HealthScoreResult> healthScoreResultList = healthScoreResultMap.get(HealthCheckDimensionEnum.CLUSTER.getDimension());
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_CLUSTER, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_CLUSTER, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_CLUSTER, 0.0f);
|
||||
|
||||
healthScore += 0;
|
||||
healthCheckPassed += 0;
|
||||
healthCheckTotal += 0;
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_CLUSTER, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_CLUSTER, this.getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_CLUSTER, Float.valueOf(healthScoreResultList.size()));
|
||||
|
||||
healthScore += this.getAllHealthScore(healthScoreResultList);
|
||||
healthCheckPassed += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_PASSED_CLUSTER);
|
||||
healthCheckTotal += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_CLUSTER);
|
||||
}
|
||||
|
||||
// broker维度
|
||||
healthScoreResultList = healthScoreResultMap.get(HealthCheckDimensionEnum.BROKER.getDimension());
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_BROKERS, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS, 0.0f);
|
||||
|
||||
healthScore += 0;
|
||||
healthCheckPassed += 0;
|
||||
healthCheckTotal += 0;
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_BROKERS, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS, this.getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS, Float.valueOf(healthScoreResultList.size()));
|
||||
|
||||
healthScore += this.getAllHealthScore(healthScoreResultList);
|
||||
healthCheckPassed += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS);
|
||||
healthCheckTotal += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS);
|
||||
}
|
||||
|
||||
|
||||
// topic维度
|
||||
healthScoreResultList = healthScoreResultMap.get(HealthCheckDimensionEnum.TOPIC.getDimension());
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_TOPICS, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS, 0.0f);
|
||||
|
||||
healthScore += 0;
|
||||
healthCheckPassed += 0;
|
||||
healthCheckTotal += 0;
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_TOPICS, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS, this.getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS, Float.valueOf(healthScoreResultList.size()));
|
||||
|
||||
healthScore += this.getAllHealthScore(healthScoreResultList);
|
||||
healthCheckPassed += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS);
|
||||
healthCheckTotal += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS);
|
||||
}
|
||||
|
||||
// group维度
|
||||
healthScoreResultList = healthScoreResultMap.get(HealthCheckDimensionEnum.GROUP.getDimension());
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_GROUPS, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS, 0.0f);
|
||||
|
||||
healthScore += 0;
|
||||
healthCheckPassed += 0;
|
||||
healthCheckTotal += 0;
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_GROUPS, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS, this.getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS, Float.valueOf(healthScoreResultList.size()));
|
||||
|
||||
healthScore += this.getAllHealthScore(healthScoreResultList);
|
||||
healthCheckPassed += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS);
|
||||
healthCheckTotal += metrics.getMetrics().get(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS);
|
||||
}
|
||||
|
||||
// 集群最终的
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE, Math.max(healthScore, Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED, healthCheckPassed);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL, healthCheckTotal);
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterMetrics calClusterTopicsHealthScore(Long clusterPhyId) {
|
||||
List<HealthScoreResult> healthScoreResultList = this.getDimensionHealthScoreResult(clusterPhyId, HealthCheckDimensionEnum.TOPIC);
|
||||
|
||||
ClusterMetrics metrics = new ClusterMetrics(clusterPhyId);
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_TOPICS, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS, 0.0f);
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_TOPICS, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS, getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS, Float.valueOf(healthScoreResultList.size()));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterMetrics calClusterBrokersHealthScore(Long clusterPhyId) {
|
||||
List<HealthScoreResult> healthScoreResultList = this.getDimensionHealthScoreResult(clusterPhyId, HealthCheckDimensionEnum.BROKER);
|
||||
|
||||
ClusterMetrics metrics = new ClusterMetrics(clusterPhyId);
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_BROKERS, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS, 0.0f);
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_BROKERS, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS, getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS, Float.valueOf(healthScoreResultList.size()));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusterMetrics calClusterGroupsHealthScore(Long clusterPhyId) {
|
||||
List<HealthScoreResult> healthScoreResultList = this.getDimensionHealthScoreResult(clusterPhyId, HealthCheckDimensionEnum.GROUP);
|
||||
|
||||
ClusterMetrics metrics = new ClusterMetrics(clusterPhyId);
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_GROUPS, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS, 0.0f);
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS, 0.0f);
|
||||
} else {
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_SCORE_GROUPS, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS, this.getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS, Float.valueOf(healthScoreResultList.size()));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TopicMetrics calTopicHealthScore(Long clusterPhyId, String topicName) {
|
||||
List<HealthScoreResult> healthScoreResultList = this.getResHealthScoreResult(clusterPhyId, HealthCheckDimensionEnum.TOPIC.getDimension(), topicName);
|
||||
|
||||
TopicMetrics metrics = new TopicMetrics(topicName, clusterPhyId,true);
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(TOPIC_METRIC_HEALTH_SCORE, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(TOPIC_METRIC_HEALTH_CHECK_PASSED, 0.0f);
|
||||
metrics.getMetrics().put(TOPIC_METRIC_HEALTH_CHECK_TOTAL, 0.0f);
|
||||
} else {
|
||||
metrics.getMetrics().put(TOPIC_METRIC_HEALTH_SCORE, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(TOPIC_METRIC_HEALTH_CHECK_PASSED, getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(TOPIC_METRIC_HEALTH_CHECK_TOTAL, Float.valueOf(healthScoreResultList.size()));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BrokerMetrics calBrokerHealthScore(Long clusterPhyId, Integer brokerId) {
|
||||
List<HealthScoreResult> healthScoreResultList = this.getResHealthScoreResult(clusterPhyId, HealthCheckDimensionEnum.BROKER.getDimension(), String.valueOf(brokerId));
|
||||
|
||||
BrokerMetrics metrics = new BrokerMetrics(clusterPhyId, brokerId);
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(BROKER_METRIC_HEALTH_SCORE, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(BROKER_METRIC_HEALTH_CHECK_PASSED, 0.0f);
|
||||
metrics.getMetrics().put(BROKER_METRIC_HEALTH_CHECK_TOTAL, 0.0f);
|
||||
} else {
|
||||
metrics.getMetrics().put(BROKER_METRIC_HEALTH_SCORE, Math.max(this.getDimensionHealthScore(healthScoreResultList), Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(BROKER_METRIC_HEALTH_CHECK_PASSED, getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(BROKER_METRIC_HEALTH_CHECK_TOTAL, Float.valueOf(healthScoreResultList.size()));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GroupMetrics calGroupHealthScore(Long clusterPhyId, String groupName) {
|
||||
List<HealthScoreResult> healthScoreResultList = this.getResHealthScoreResult(clusterPhyId, HealthCheckDimensionEnum.GROUP.getDimension(), groupName);
|
||||
|
||||
GroupMetrics metrics = new GroupMetrics(clusterPhyId, groupName, true);
|
||||
if (ValidateUtils.isEmptyList(healthScoreResultList)) {
|
||||
metrics.getMetrics().put(GROUP_METRIC_HEALTH_SCORE, Constant.MIN_HEALTH_SCORE);
|
||||
metrics.getMetrics().put(GROUP_METRIC_HEALTH_CHECK_PASSED, 0.0f);
|
||||
metrics.getMetrics().put(GROUP_METRIC_HEALTH_CHECK_TOTAL, 0.0f);
|
||||
} else {
|
||||
metrics.getMetrics().put(GROUP_METRIC_HEALTH_SCORE, Math.max(this.getDimensionHealthScore(healthScoreResultList),Constant.MIN_HEALTH_SCORE));
|
||||
metrics.getMetrics().put(GROUP_METRIC_HEALTH_CHECK_PASSED, getHealthCheckPassed(healthScoreResultList));
|
||||
metrics.getMetrics().put(GROUP_METRIC_HEALTH_CHECK_TOTAL, Float.valueOf(healthScoreResultList.size()));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HealthScoreResult> getClusterHealthScoreResult(Long clusterPhyId) {
|
||||
List<HealthCheckResultPO> poList = healthCheckResultService.getClusterHealthCheckResult(clusterPhyId);
|
||||
|
||||
// <检查项，<检查结果>>
|
||||
Map<String, List<HealthCheckResultPO>> checkResultMap = new HashMap<>();
|
||||
for (HealthCheckResultPO po: poList) {
|
||||
checkResultMap.putIfAbsent(po.getConfigName(), new ArrayList<>());
|
||||
checkResultMap.get(po.getConfigName()).add(po);
|
||||
}
|
||||
|
||||
// 每个维度的权重和
|
||||
Map<String, Float> dimensionTotalWeightMap = new HashMap<>();
|
||||
Float allDimensionTotalWeight = 0f;
|
||||
|
||||
Map<String, BaseClusterHealthConfig> configMap = healthCheckResultService.getClusterHealthConfig(clusterPhyId);
|
||||
for (HealthCheckNameEnum nameEnum: HealthCheckNameEnum.values()) {
|
||||
BaseClusterHealthConfig baseConfig = configMap.get(nameEnum.getConfigName());
|
||||
if (baseConfig == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
allDimensionTotalWeight += baseConfig.getWeight();
|
||||
|
||||
Float totalWeight = dimensionTotalWeightMap.getOrDefault(nameEnum.getDimensionEnum().name(), 0f);
|
||||
dimensionTotalWeightMap.put(nameEnum.getDimensionEnum().name(), totalWeight + baseConfig.getWeight());
|
||||
}
|
||||
|
||||
List<HealthScoreResult> healthScoreResultList = new ArrayList<>();
|
||||
for (HealthCheckNameEnum nameEnum: HealthCheckNameEnum.values()) {
|
||||
BaseClusterHealthConfig baseConfig = configMap.get(nameEnum.getConfigName());
|
||||
if (baseConfig == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
healthScoreResultList.add(new HealthScoreResult(
|
||||
nameEnum,
|
||||
dimensionTotalWeightMap.getOrDefault(nameEnum.getDimensionEnum().name(), 0f),
|
||||
allDimensionTotalWeight,
|
||||
baseConfig,
|
||||
checkResultMap.getOrDefault(nameEnum.getConfigName(), new ArrayList<>()))
|
||||
);
|
||||
}
|
||||
|
||||
return healthScoreResultList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HealthScoreResult> getDimensionHealthScoreResult(Long clusterPhyId, HealthCheckDimensionEnum dimensionEnum) {
|
||||
List<HealthCheckResultPO> poList = healthCheckResultService.getClusterResourcesHealthCheckResult(clusterPhyId, dimensionEnum.getDimension());
|
||||
|
||||
// <检查项，<通过的数量，不通过的数量>>
|
||||
Map<String, List<HealthCheckResultPO>> checkResultMap = new HashMap<>();
|
||||
for (HealthCheckResultPO po: poList) {
|
||||
checkResultMap.putIfAbsent(po.getConfigName(), new ArrayList<>());
|
||||
checkResultMap.get(po.getConfigName()).add(po);
|
||||
}
|
||||
|
||||
// 每个维度的权重和
|
||||
Float totalWeight = 0f;
|
||||
|
||||
Map<String, BaseClusterHealthConfig> configMap = healthCheckResultService.getClusterHealthConfig(clusterPhyId);
|
||||
for (HealthCheckNameEnum nameEnum: HealthCheckNameEnum.getByDimension(dimensionEnum)) {
|
||||
BaseClusterHealthConfig baseConfig = configMap.get(nameEnum.getConfigName());
|
||||
if (baseConfig == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
totalWeight += baseConfig.getWeight();
|
||||
}
|
||||
|
||||
Float allDimensionTotalWeight = configMap.values().stream().map(elem -> elem.getWeight()).reduce(Float::sum).get();
|
||||
|
||||
List<HealthScoreResult> healthScoreResultList = new ArrayList<>();
|
||||
for (HealthCheckNameEnum nameEnum: HealthCheckNameEnum.getByDimension(dimensionEnum)) {
|
||||
BaseClusterHealthConfig baseConfig = configMap.get(nameEnum.getConfigName());
|
||||
if (baseConfig == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
healthScoreResultList.add(new HealthScoreResult(nameEnum, totalWeight, allDimensionTotalWeight, baseConfig, checkResultMap.getOrDefault(nameEnum.getConfigName(), new ArrayList<>())));
|
||||
}
|
||||
|
||||
return healthScoreResultList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HealthScoreResult> getResHealthScoreResult(Long clusterPhyId, Integer dimension, String resNme) {
|
||||
List<HealthCheckResultPO> poList = healthCheckResultService.getClusterResourcesHealthCheckResult(clusterPhyId, dimension);
|
||||
Map<String, List<HealthCheckResultPO>> checkResultMap = new HashMap<>();
|
||||
for (HealthCheckResultPO po: poList) {
|
||||
checkResultMap.putIfAbsent(po.getConfigName(), new ArrayList<>());
|
||||
checkResultMap.get(po.getConfigName()).add(po);
|
||||
}
|
||||
|
||||
// 每个维度的权重和
|
||||
Float totalWeight = 0f;
|
||||
|
||||
Map<String, BaseClusterHealthConfig> configMap = healthCheckResultService.getClusterHealthConfig(clusterPhyId);
|
||||
for (HealthCheckNameEnum nameEnum: HealthCheckNameEnum.getByDimensionCode(dimension)) {
|
||||
BaseClusterHealthConfig baseConfig = configMap.get(nameEnum.getConfigName());
|
||||
if (baseConfig == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
totalWeight += baseConfig.getWeight();
|
||||
}
|
||||
|
||||
List<HealthScoreResult> healthScoreResultList = new ArrayList<>();
|
||||
for (HealthCheckNameEnum nameEnum: HealthCheckNameEnum.getByDimensionCode(dimension)) {
|
||||
BaseClusterHealthConfig baseConfig = configMap.get(nameEnum.getConfigName());
|
||||
if (baseConfig == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
healthScoreResultList.add(new HealthScoreResult(nameEnum, totalWeight, null, baseConfig, checkResultMap.getOrDefault(nameEnum.getConfigName(), new ArrayList<>())));
|
||||
}
|
||||
|
||||
return healthScoreResultList;
|
||||
}
|
||||
|
||||
private float getAllHealthScore(List<HealthScoreResult> healthScoreResultList){
|
||||
if(CollectionUtils.isEmpty(healthScoreResultList)){return 0f;}
|
||||
|
||||
return healthScoreResultList.stream().map(elem -> elem.calAllWeightHealthScore()).reduce(Float::sum).get();
|
||||
}
|
||||
|
||||
private float getDimensionHealthScore(List<HealthScoreResult> healthScoreResultList){
|
||||
if(CollectionUtils.isEmpty(healthScoreResultList)){return 0f;}
|
||||
|
||||
return healthScoreResultList.stream().map(elem -> elem.calDimensionWeightHealthScore()).reduce(Float::sum).get();
|
||||
}
|
||||
|
||||
private float getHealthCheckPassed(List<HealthScoreResult> healthScoreResultList){
|
||||
if(CollectionUtils.isEmpty(healthScoreResultList)){return 0f;}
|
||||
|
||||
return Float.valueOf(healthScoreResultList.stream().filter(elem -> elem.getPassed()).count());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.job;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.JobStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobModifyDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobPartitionDetailVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.component.BaseHandle;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobActionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author d06679
|
||||
* @date 2020/12/21
|
||||
*/
|
||||
public interface JobHandler extends BaseHandle {
|
||||
|
||||
JobTypeEnum type();
|
||||
|
||||
/**
|
||||
* 创建一个任务
|
||||
* 1、校验任务内容是否合法
|
||||
* 2、提交任务，但是不执行
|
||||
* @param job 任务数据
|
||||
* @return result
|
||||
*/
|
||||
Result<Void> submit(Job job, String operator);
|
||||
|
||||
/**
|
||||
* 创建一个任务
|
||||
*
|
||||
* 1、校验任务内容是否合法
|
||||
* 2、提交任务
|
||||
* @param job 任务数据
|
||||
* @return result
|
||||
*/
|
||||
Result<Void> delete(Job job, String operator);
|
||||
|
||||
/**
|
||||
* 修改一个任务
|
||||
* 1、校验任务内容是否合法
|
||||
* 2、提交任务
|
||||
* @param job 任务数据
|
||||
* @return result
|
||||
*/
|
||||
Result<Void> modify(Job job, String operator);
|
||||
|
||||
/**
|
||||
* 修改一个任务的限流值
|
||||
* @param job 任务数据
|
||||
* @param limit 限流值
|
||||
* @return result
|
||||
*/
|
||||
Result<Void> updateLimit(Job job, Long limit, String operator);
|
||||
|
||||
/**
|
||||
* 处理任务
|
||||
* @param job 任务
|
||||
* @return result
|
||||
*/
|
||||
Result<Void> process(Job job, JobActionEnum action, String operator);
|
||||
|
||||
/**
|
||||
* 获取任务详细状态
|
||||
* @param job job
|
||||
* @return JobStatus
|
||||
*/
|
||||
Result<JobStatus> status(Job job);
|
||||
|
||||
/**
|
||||
* 获取任务详细信息
|
||||
* @param job job
|
||||
* @return AbstractTaskDetail
|
||||
*/
|
||||
Result<JobDetail> getTaskDetail(Job job);
|
||||
|
||||
/**
|
||||
* 获取任务详细信息
|
||||
* @param job job
|
||||
* @return AbstractTaskDetail
|
||||
*/
|
||||
Result<JobModifyDetail> getTaskModifyDetail(Job job);
|
||||
|
||||
/**
|
||||
* 获取任务详细信息
|
||||
* @param job job
|
||||
* @return AbstractTaskDetail
|
||||
*/
|
||||
Result<List<SubJobPartitionDetailVO>> getSubJobPartitionDetail(Job job, String topic);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.job;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.job.JobDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.job.JobPaginationDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobPartitionDetailVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 任务 Service
|
||||
*
|
||||
* @author d06679
|
||||
* @date 2020/12/21
|
||||
*/
|
||||
public interface JobService {
|
||||
|
||||
/**
|
||||
* 提交一个任务
|
||||
* @param jobDTO 任务数据
|
||||
* @return Result
|
||||
* @throws AdminOperateException 异常
|
||||
*/
|
||||
Result<Void> addTask(Long clusterPhyId, JobDTO jobDTO, String operator);
|
||||
|
||||
/**
|
||||
* 删除一个任务
|
||||
* @param jobId
|
||||
* @param operator
|
||||
* @return
|
||||
*/
|
||||
Result<Void> deleteById(Long clusterPhyId, Long jobId, String operator);
|
||||
|
||||
/**
|
||||
* 通过id更新任务
|
||||
* @param task task
|
||||
* @return int
|
||||
*/
|
||||
Result<Void> updateTask(Long clusterPhyId, JobDTO task, String operator);
|
||||
|
||||
/**
|
||||
* 通过id获取任务
|
||||
*
|
||||
* @param id 任务id
|
||||
* @return TaskPO
|
||||
*/
|
||||
Job getById(Long clusterPhyId, Long id);
|
||||
|
||||
/**
|
||||
* 通过集群id和type获取任务
|
||||
*
|
||||
* @param type 类型
|
||||
* @return clusterPhyId 集群id
|
||||
*/
|
||||
Job getByClusterIdAndType(Long clusterPhyId, Integer type);
|
||||
|
||||
/**
|
||||
* 获取 job 的详细信息
|
||||
* @param jobId
|
||||
* @return
|
||||
*/
|
||||
Result<JobDetailVO> getJobDetail(Long clusterPhyId, Long jobId);
|
||||
|
||||
/**
|
||||
* 获取 job 的详细信息
|
||||
* @param jobId
|
||||
* @return
|
||||
*/
|
||||
Result<JobModifyDetailVO> getJobModifyDetail(Long clusterPhyId, Long jobId);
|
||||
|
||||
/**
|
||||
* 获取 job 相关的子任务的 partition 流量信息
|
||||
* @param jobId
|
||||
* @return
|
||||
*/
|
||||
Result<List<SubJobPartitionDetailVO>> getSubJobPartitionDetail(Long clusterPhyId, Long jobId, String topic);
|
||||
|
||||
/**
|
||||
* 获取 job 相关的 broker 节点流量信息
|
||||
* @param jobId
|
||||
* @return
|
||||
*/
|
||||
Result<List<JobTrafficBrokerVO>> getJobNodeTraffic(Long clusterPhyId, Long jobId);
|
||||
|
||||
/**
|
||||
* 分页查询
|
||||
* @param clusterPhyId
|
||||
* @return
|
||||
*/
|
||||
PaginationResult<JobOverViewVO> pagingJobs(Long clusterPhyId, JobPaginationDTO dto);
|
||||
|
||||
/**
|
||||
* 获取 job 模块的状态统计信息
|
||||
* @param clusterPhyId
|
||||
* @return
|
||||
*/
|
||||
Result<JobStateVO> state(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 根据某个 job 的限流值
|
||||
* @param clusterPhyId
|
||||
* @param jobId
|
||||
* @param limit
|
||||
* @return
|
||||
*/
|
||||
Result<Void> updateJobTrafficLimit(Long clusterPhyId, Long jobId, Long limit, String operator);
|
||||
|
||||
/**
|
||||
* 定时检查集群中是否有需要执行和正在执行的任务
|
||||
* @param clusterPhyId
|
||||
* @return
|
||||
*/
|
||||
void scheduleJobByClusterId(Long clusterPhyId);
|
||||
|
||||
Integer countJobsByCluster(Long clusterPhyId);
|
||||
|
||||
Integer countJobsByClusterAndJobStatus(Long clusterPhyId, Integer jobStatus);
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.job.handler;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.reassign.plan.ReassignTopicPlanDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.JobStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.content.JobCommunityReassignContent;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobModifyDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReassignJobDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.detail.ReassignJobDetailDataGroupByTopic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.reassign.ReassignSubJobPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobPartitionDetailVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.ReassignConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobActionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.job.JobHandler;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignJobService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.TopicMetricVersionItems;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 通用Job 和 具体的ReassignJob 的中间层，将 job 的相关操作，转到 reassignJob 中
|
||||
*/
|
||||
public abstract class AbstractReassignJobHandler implements JobHandler {
|
||||
@Autowired
|
||||
private ReassignJobService reassignJobService;
|
||||
|
||||
@Autowired
|
||||
private TopicMetricService topicMetricService;
|
||||
|
||||
@Override
|
||||
public Result<Void> submit(Job job, String operator) {
|
||||
// 获取任务详情信息
|
||||
JobCommunityReassignContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobCommunityReassignContent.class);
|
||||
if (dto == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "jobData格式错误");
|
||||
}
|
||||
|
||||
// 转换格式，并创建任务
|
||||
return reassignJobService.create(job.getId(), ReassignConverter.convert2ReplaceReassignJob(dto), operator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> delete(Job job, String operator) {
|
||||
return reassignJobService.delete(job.getId(), operator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> modify(Job job, String operator) {
|
||||
// 获取任务详情信息
|
||||
JobCommunityReassignContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobCommunityReassignContent.class);
|
||||
|
||||
// 修改任务
|
||||
return reassignJobService.modify(job.getId(), ReassignConverter.convert2ReplaceReassignJob(dto), operator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<JobModifyDetail> getTaskModifyDetail(Job job) {
|
||||
// 获取任务详情信息
|
||||
JobCommunityReassignContent dto = ConvertUtil.str2ObjByJson(job.getJobData(), JobCommunityReassignContent.class);
|
||||
if (dto == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "jobData格式错误");
|
||||
}
|
||||
|
||||
JobModifyDetail detail = ConvertUtil.obj2Obj(job, JobModifyDetail.class);
|
||||
detail.setJobData(ConvertUtil.obj2JsonWithIgnoreCircularReferenceDetect(this.updateLatestMetrics(dto)));
|
||||
|
||||
return Result.buildSuc(detail);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> updateLimit(Job job, Long limit, String operator){
|
||||
return reassignJobService.modifyThrottle(job.getId(), limit, operator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> process(Job job, JobActionEnum action, String operator) {
|
||||
if (JobActionEnum.START.equals(action)) {
|
||||
return reassignJobService.execute(job.getId(), operator);
|
||||
}
|
||||
|
||||
if (JobActionEnum.CANCEL.equals(action)) {
|
||||
return reassignJobService.cancel(job.getId(), operator);
|
||||
}
|
||||
|
||||
// 迁移中，不支持该操作
|
||||
return Result.buildFromRSAndMsg(ResultStatus.OPERATION_FORBIDDEN, String.format("不支持[%s]操作", action.getValue()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<JobStatus> status(Job job) {
|
||||
// Topic下每个分区的状态
|
||||
Map<String, List<ReassignSubJobPO>> topicJobsMap = new HashMap<>();
|
||||
|
||||
// 获取子任务，并按照Topic进行聚合
|
||||
List<ReassignSubJobPO> allSubJobPOList = reassignJobService.getSubJobsByJobId(job.getId());
|
||||
allSubJobPOList.forEach(elem -> {
|
||||
topicJobsMap.putIfAbsent(elem.getTopicName(), new ArrayList<>());
|
||||
topicJobsMap.get(elem.getTopicName()).add(elem);
|
||||
});
|
||||
|
||||
// 获取每个Topic的状态
|
||||
List<Integer> topicStatusList = new ArrayList<>();
|
||||
for (List<ReassignSubJobPO> topicJobPOList: topicJobsMap.values()) {
|
||||
topicStatusList.add(new JobStatus(
|
||||
topicJobPOList.stream().map(elem -> elem.getStatus()).collect(Collectors.toList())
|
||||
).getStatus());
|
||||
}
|
||||
|
||||
// 聚合Topic的结果
|
||||
return Result.buildSuc(new JobStatus(topicStatusList));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<JobDetail> getTaskDetail(Job job) {
|
||||
Result<ReassignJobDetail> detailResult = reassignJobService.getJobDetailsGroupByTopic(job.getId());
|
||||
if (detailResult.failed()) {
|
||||
return Result.buildFromIgnoreData(detailResult);
|
||||
}
|
||||
|
||||
return Result.buildSuc(ReassignConverter.convert2JobDetail(job, detailResult.getData()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<SubJobPartitionDetailVO>> getSubJobPartitionDetail(Job job, String topic) {
|
||||
Result<ReassignJobDetail> detailResult = reassignJobService.getJobDetailsGroupByTopic(job.getId());
|
||||
if (detailResult.failed()) {
|
||||
return Result.buildFromIgnoreData(detailResult);
|
||||
}
|
||||
|
||||
List<ReassignJobDetailDataGroupByTopic> detailDataGroupByTopicList = detailResult.getData().getReassignTopicDetailsList()
|
||||
.stream()
|
||||
.filter(elem -> elem.getTopicName().equals(topic))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (detailDataGroupByTopicList.isEmpty()) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(job.getClusterId(), topic));
|
||||
}
|
||||
|
||||
return Result.buildSuc(ReassignConverter.convert2SubJobPartitionDetailVOList(detailDataGroupByTopicList.get(0)));
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private JobCommunityReassignContent updateLatestMetrics(JobCommunityReassignContent dto) {
|
||||
List<String> topicNameList = dto.getTopicPlanList().stream().map(elem -> elem.getTopicName()).collect(Collectors.toList());
|
||||
|
||||
// 清空历史数据
|
||||
for (ReassignTopicPlanDTO elem: dto.getTopicPlanList()) {
|
||||
elem.setLatestDaysAvgBytesInList(new ArrayList<>());
|
||||
elem.setLatestDaysMaxBytesInList(new ArrayList<>());
|
||||
}
|
||||
|
||||
Long now = System.currentTimeMillis();
|
||||
// 补充近三天指标
|
||||
for (int idx = 0; idx < 3; ++idx) {
|
||||
Long startTime = now - (idx + 1) * 24L * 60L * 60L * 1000L;
|
||||
Long endTime = now - idx * 24L * 60L * 60L * 1000L;
|
||||
|
||||
// 查询avg指标
|
||||
Result<Map<String, MetricPointVO>> avgMetricMapResult = topicMetricService.getAggMetricPointFromES(
|
||||
dto.getClusterId(),
|
||||
topicNameList,
|
||||
TopicMetricVersionItems.TOPIC_METRIC_BYTES_IN,
|
||||
AggTypeEnum.AVG,
|
||||
startTime,
|
||||
endTime
|
||||
);
|
||||
Map<String, MetricPointVO> avgMetricMap = avgMetricMapResult.hasData()? avgMetricMapResult.getData(): new HashMap<>();
|
||||
avgMetricMap.values().forEach(elem -> elem.setTimeStamp(endTime));
|
||||
|
||||
// 查询max指标
|
||||
Result<Map<String, MetricPointVO>> maxMetricMapResult = topicMetricService.getAggMetricPointFromES(
|
||||
dto.getClusterId(),
|
||||
topicNameList,
|
||||
TopicMetricVersionItems.TOPIC_METRIC_BYTES_IN,
|
||||
AggTypeEnum.MAX,
|
||||
startTime,
|
||||
endTime
|
||||
);
|
||||
Map<String, MetricPointVO> maxMetricMap = maxMetricMapResult.hasData()? maxMetricMapResult.getData(): new HashMap<>();
|
||||
|
||||
// 补充指标信息
|
||||
for (ReassignTopicPlanDTO elem: dto.getTopicPlanList()) {
|
||||
// 设置avg
|
||||
elem.getLatestDaysAvgBytesInList().add(avgMetricMap.get(elem.getTopicName()));
|
||||
|
||||
// 设置max
|
||||
elem.getLatestDaysMaxBytesInList().add(maxMetricMap.get(elem.getTopicName()));
|
||||
}
|
||||
}
|
||||
|
||||
return dto;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.job.handler;
|
||||
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.JobConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component(JobConstant.TOPIC_REPLICA_MOVE)
|
||||
public class TopicReplicaMoveJobHandler extends AbstractReassignJobHandler {
|
||||
|
||||
@Override
|
||||
public JobTypeEnum type() {
|
||||
return JobTypeEnum.TOPIC_REPLICA_MOVE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.job.handler;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.JobConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component(JobConstant.TOPIC_REPLICA_SCALA)
|
||||
public class TopicReplicaScalaJobHandler extends AbstractReassignJobHandler {
|
||||
@Override
|
||||
public JobTypeEnum type() {
|
||||
return JobTypeEnum.TOPIC_REPLICA_SCALA;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.job.impl;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.job.JobDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.job.JobPaginationDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.Job;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.JobStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.job.detail.JobModifyDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BrokerMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.job.JobPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubBrokerJobVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobPartitionDetailVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.job.sub.SubJobVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.component.HandleFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobActionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobHandleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.job.JobHandler;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.job.JobService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.job.JobDAO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.interceptor.TransactionAspectSupport;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.BrokerMetricVersionItems.*;
|
||||
|
||||
@Service
|
||||
public class JobServiceImpl implements JobService {
|
||||
private static final ILog LOGGER = LogFactory.getLog(JobServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private JobDAO jobDao;
|
||||
|
||||
@Autowired
|
||||
private HandleFactory handleFactory;
|
||||
|
||||
@Autowired
|
||||
private BrokerMetricService brokerMetricService;
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Result<Void> addTask(Long clusterPhyId, JobDTO jobDTO, String operator) {
|
||||
JobTypeEnum typeEnum = JobTypeEnum.valueOfType(jobDTO.getJobType());
|
||||
if(JobTypeEnum.UNKNOWN.equals(typeEnum)){
|
||||
return Result.buildFailure("任务类型不存在");
|
||||
}
|
||||
|
||||
Job job = ConvertUtil.obj2Obj(jobDTO, Job.class);
|
||||
job.setCreator(operator);
|
||||
job.setClusterId(clusterPhyId);
|
||||
|
||||
try {
|
||||
// 写入job表
|
||||
this.insert(job);
|
||||
|
||||
JobHandler handler = getJobHandlerByType(jobDTO.getJobType());
|
||||
Result<Void> handlerRet = handler.submit(job, operator);
|
||||
if (handlerRet.failed()) {
|
||||
// 如果这一步出错了，则对上一步进行手动回滚
|
||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
||||
}
|
||||
|
||||
return handlerRet;
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=addTask||clusterPhyId={}||jobDTO={}||operator={}||errMsg=exception.", clusterPhyId, jobDTO, operator, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(OPERATION_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> deleteById(Long clusterPhyId, Long jobId, String operator) {
|
||||
JobPO jobPO = jobDao.selectById(jobId);
|
||||
if(null == jobPO){return Result.buildFailure(NOT_EXIST);}
|
||||
|
||||
JobHandler handler = getJobHandlerByType(jobPO.getJobType());
|
||||
|
||||
Result<Void> ret = handler.delete(ConvertUtil.obj2Obj(jobPO, Job.class), operator);
|
||||
if(null != ret && ret.successful()){
|
||||
return Result.build(jobDao.deleteById(jobId) > 0);
|
||||
}
|
||||
|
||||
return Result.buildFromIgnoreData(ret);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> updateTask(Long clusterPhyId, JobDTO task, String operator) {
|
||||
Job job = getById(clusterPhyId, task.getId());
|
||||
if(null == job){return Result.buildFailure(NOT_EXIST);}
|
||||
|
||||
if(JobStatusEnum.RUNNING.getStatus() == job.getJobStatus()){
|
||||
return Result.buildFailure("正在运行中的任务无法编辑！");
|
||||
}
|
||||
|
||||
if(!StringUtils.isEmpty(task.getJobDesc())){
|
||||
job.setJobDesc(task.getJobDesc());
|
||||
}
|
||||
if(!StringUtils.isEmpty(task.getJobData())){
|
||||
job.setJobData(task.getJobData());
|
||||
}
|
||||
if(null != task.getPlanTime()){
|
||||
job.setPlanTime(task.getPlanTime());
|
||||
}
|
||||
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
Result<Void> handlerRet = jobHandler.modify(job, operator);
|
||||
if(null != handlerRet && handlerRet.successful()){
|
||||
JobPO jobPO = ConvertUtil.obj2Obj(task, JobPO.class);
|
||||
jobPO.setUpdateTime(new Date());
|
||||
return Result.build(jobDao.updateById(jobPO) > 0);
|
||||
}
|
||||
|
||||
return Result.buildFailure(OPERATION_FAILED);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Job getById(Long clusterPhyId, Long id) {
|
||||
JobPO jobPO = jobDao.selectById(id);
|
||||
return ConvertUtil.obj2Obj(jobPO, Job.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Job getByClusterIdAndType(Long clusterPhyId, Integer type) {
|
||||
JobPO queryParam = new JobPO();
|
||||
queryParam.setClusterId(clusterPhyId);
|
||||
queryParam.setJobType(type);
|
||||
JobPO jobPO = jobDao.selectOne(new QueryWrapper(queryParam));
|
||||
if (jobPO == null){
|
||||
return null;
|
||||
}
|
||||
return ConvertUtil.obj2Obj(jobPO, Job.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<JobDetailVO> getJobDetail(Long clusterPhyId, Long jobId) {
|
||||
Job job = this.getById(clusterPhyId, jobId);
|
||||
if(null == job){
|
||||
return Result.buildFailure(NOT_EXIST);
|
||||
}
|
||||
|
||||
JobHandler jobHandler = this.getJobHandlerByType(job.getJobType());
|
||||
Result<JobDetail> jobDetailResult = jobHandler.getTaskDetail(job);
|
||||
if (jobDetailResult.failed()) {
|
||||
return Result.buildFromIgnoreData(jobDetailResult);
|
||||
}
|
||||
|
||||
return Result.buildSuc(ConvertUtil.obj2Obj(jobDetailResult.getData(), JobDetailVO.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<JobModifyDetailVO> getJobModifyDetail(Long clusterPhyId, Long jobId) {
|
||||
Job job = getById(clusterPhyId, jobId);
|
||||
if(null == job){
|
||||
return Result.buildFailure(NOT_EXIST);
|
||||
}
|
||||
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
Result<JobModifyDetail> jobDetailResult = jobHandler.getTaskModifyDetail(job);
|
||||
if (jobDetailResult.failed()) {
|
||||
return Result.buildFromIgnoreData(jobDetailResult);
|
||||
}
|
||||
|
||||
return Result.buildSuc(ConvertUtil.obj2Obj(jobDetailResult.getData(), JobModifyDetailVO.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<SubJobPartitionDetailVO>> getSubJobPartitionDetail(Long clusterPhyId, Long jobId, String topic) {
|
||||
Job job = this.getById(clusterPhyId, jobId);
|
||||
if(null == job){
|
||||
return Result.buildFailure(NOT_EXIST);
|
||||
}
|
||||
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
Result<List<SubJobPartitionDetailVO>> subJobPartitionDetailRet = jobHandler.getSubJobPartitionDetail(job, topic);
|
||||
if (subJobPartitionDetailRet.failed()) {
|
||||
return Result.buildFromIgnoreData(subJobPartitionDetailRet);
|
||||
}
|
||||
|
||||
return Result.buildSuc(ConvertUtil.list2List(subJobPartitionDetailRet.getData(), SubJobPartitionDetailVO.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<JobTrafficBrokerVO>> getJobNodeTraffic(Long clusterPhyId, Long jobId) {
|
||||
Result<JobDetailVO> jobDetailVORet = this.getJobDetail(clusterPhyId, jobId);
|
||||
if(null == jobDetailVORet || jobDetailVORet.failed()){
|
||||
return Result.buildFail();
|
||||
}
|
||||
|
||||
//1、获取任务的详情
|
||||
JobDetailVO jobDetailVO = jobDetailVORet.getData();
|
||||
if(null == jobDetailVO || CollectionUtils.isEmpty(jobDetailVO.getSubJobs())){
|
||||
return Result.buildFail();
|
||||
}
|
||||
|
||||
Map<Integer, Tuple<List<Integer>/*source*/, List<Integer>/*des*/>> brokerSourceDesMap = new HashMap<>();
|
||||
Set<Integer> allBrokerIdSet = new HashSet<>();
|
||||
|
||||
//2、从任务详情中获取到某个 broker 的流入流出目标 broker 列表
|
||||
for(SubJobVO subJobVO : jobDetailVO.getSubJobs()){
|
||||
SubBrokerJobVO subBrokerJobVO = (SubBrokerJobVO)subJobVO;
|
||||
|
||||
List<Integer> desBrokerIds = subBrokerJobVO.getDesBrokers();
|
||||
List<Integer> sourceBrokerIds = subBrokerJobVO.getSourceBrokers();
|
||||
|
||||
allBrokerIdSet.addAll(desBrokerIds);
|
||||
allBrokerIdSet.addAll(sourceBrokerIds);
|
||||
|
||||
brokerSourceDesMap = genBrokerSourceDesInfo(brokerSourceDesMap, desBrokerIds, sourceBrokerIds);
|
||||
}
|
||||
|
||||
//3、获取brokerId、brokerHost
|
||||
Map<Integer, String> brokerIdHostMap = genBrokerIdHostMap(clusterPhyId, allBrokerIdSet);
|
||||
|
||||
//4、获取所有 allBrokerIdSet 的指标信息
|
||||
Result<List<BrokerMetrics>> brokerMetricsRet = brokerMetricService.getLatestMetricsFromES(
|
||||
clusterPhyId, new ArrayList<>(allBrokerIdSet));
|
||||
|
||||
if(null == brokerMetricsRet || brokerMetricsRet.failed() || CollectionUtils.isEmpty(brokerMetricsRet.getData())){
|
||||
return Result.buildFail();
|
||||
}
|
||||
|
||||
//5、构建 JobTrafficBrokerVO 列表
|
||||
Map<Integer, Tuple<List<Integer>, List<Integer>>> finalBrokerSourceDesMap = brokerSourceDesMap;
|
||||
List<JobTrafficBrokerVO> jobTrafficBrokerVOS = brokerMetricsRet.getData().stream().map( b -> {
|
||||
JobTrafficBrokerVO jobTrafficBrokerVO = new JobTrafficBrokerVO();
|
||||
Integer brokerId = b.getBrokerId();
|
||||
jobTrafficBrokerVO.setId(jobId);
|
||||
jobTrafficBrokerVO.setBrokerId(brokerId);
|
||||
jobTrafficBrokerVO.setBrokerHost(brokerIdHostMap.get(brokerId));
|
||||
jobTrafficBrokerVO.setByteInTotal(b.getMetrics().get(BROKER_METRIC_BYTES_IN) != null?Double.valueOf(b.getMetrics().get(BROKER_METRIC_BYTES_IN)):0D);
|
||||
jobTrafficBrokerVO.setByteOutTotal(b.getMetrics().get(BROKER_METRIC_BYTES_OUT) != null?Double.valueOf(b.getMetrics().get(BROKER_METRIC_BYTES_OUT)):0D);
|
||||
jobTrafficBrokerVO.setByteInJob(b.getMetrics().get(BROKER_METRIC_REASSIGNMENT_BYTES_IN) != null?Double.valueOf(b.getMetrics().get(BROKER_METRIC_REASSIGNMENT_BYTES_IN)):0D);
|
||||
jobTrafficBrokerVO.setByteOutJob(b.getMetrics().get(BROKER_METRIC_REASSIGNMENT_BYTES_OUT) !=null?Double.valueOf(b.getMetrics().get(BROKER_METRIC_REASSIGNMENT_BYTES_OUT)):0D);
|
||||
jobTrafficBrokerVO.setInBrokers(getBrokerIdHostMap(brokerIdHostMap, finalBrokerSourceDesMap.get(brokerId).getV1()));
|
||||
jobTrafficBrokerVO.setOutBrokers(getBrokerIdHostMap(brokerIdHostMap, finalBrokerSourceDesMap.get(brokerId).getV2()));
|
||||
return jobTrafficBrokerVO;
|
||||
} ).collect( Collectors.toList());
|
||||
|
||||
return Result.buildSuc(jobTrafficBrokerVOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<JobOverViewVO> pagingJobs(Long clusterPhyId, JobPaginationDTO dto) {
|
||||
LambdaQueryWrapper<JobPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(JobPO::getClusterId, clusterPhyId);
|
||||
queryWrapper.eq(-1 != dto.getType(), JobPO::getJobType, dto.getType());
|
||||
queryWrapper.like(!StringUtils.isEmpty(dto.getJobTarget()), JobPO::getTarget, dto.getJobTarget());
|
||||
queryWrapper.like(!StringUtils.isEmpty(dto.getCreator()), JobPO::getCreator, dto.getCreator());
|
||||
queryWrapper.in(!CollectionUtils.isEmpty(dto.getStatus()), JobPO::getJobStatus, dto.getStatus());
|
||||
queryWrapper.orderByAsc(JobPO::getJobStatus);
|
||||
queryWrapper.orderByDesc(JobPO::getPlanTime);
|
||||
|
||||
IPage<JobPO> page = new Page<>(dto.getPageNo(), dto.getPageSize());
|
||||
page.setTotal(jobDao.selectCount(queryWrapper));
|
||||
|
||||
jobDao.selectPage(page, queryWrapper);
|
||||
|
||||
return PaginationResult.buildSuc(jobPOS2JobVOS(page.getRecords()), page);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<JobStateVO> state(Long clusterPhyId) {
|
||||
List<Job> jobs = listAllJobByClusterId(clusterPhyId);
|
||||
|
||||
int allNus = jobs.size();
|
||||
int doingNu = 0;
|
||||
int waitingNu = 0;
|
||||
int successNu = 0;
|
||||
int failedNu = 0;
|
||||
|
||||
for(Job job : jobs){
|
||||
if(JobStatusEnum.SUCCESS.getStatus() == job.getJobStatus()){
|
||||
successNu++;
|
||||
}else if(JobStatusEnum.RUNNING.getStatus() == job.getJobStatus()){
|
||||
doingNu++;
|
||||
}else if(JobStatusEnum.WAITING.getStatus() == job.getJobStatus()){
|
||||
waitingNu++;
|
||||
}else {
|
||||
failedNu++;
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(new JobStateVO(allNus, doingNu, waitingNu, successNu, failedNu));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> updateJobTrafficLimit(Long clusterPhyId, Long jobId, Long limit, String operator) {
|
||||
Job job = getById(clusterPhyId, jobId);
|
||||
if(null == job){
|
||||
return Result.buildFailure(NOT_EXIST);
|
||||
}
|
||||
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
return jobHandler.updateLimit(job, limit, operator);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void scheduleJobByClusterId(Long clusterPhyId) {
|
||||
startJob(clusterPhyId);
|
||||
updateStatusJob(clusterPhyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countJobsByCluster(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<JobPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(JobPO::getClusterId, clusterPhyId);
|
||||
return jobDao.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countJobsByClusterAndJobStatus(Long clusterPhyId, Integer jobStatus) {
|
||||
LambdaQueryWrapper<JobPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(JobPO::getClusterId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(JobPO::getJobStatus, jobStatus);
|
||||
return jobDao.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
private void startJob(Long clusterPhyId){
|
||||
try {
|
||||
//1、该集群还有在执行的任务则不能继续执行
|
||||
List<Job> runningJobs = this.listJobByClusterIdAndStatusSortByPlanTime(clusterPhyId, JobStatusEnum.RUNNING.getStatus());
|
||||
if(!CollectionUtils.isEmpty(runningJobs)){
|
||||
LOGGER.debug("method=startJob||clusterPhyId={}||errMsg=no running job", clusterPhyId);
|
||||
return;
|
||||
}
|
||||
|
||||
//2、获取所有等待需要执行的任务
|
||||
List<Job> waitingJobs = this.listJobByClusterIdAndStatusSortByPlanTime(clusterPhyId, JobStatusEnum.WAITING.getStatus());
|
||||
if(CollectionUtils.isEmpty(waitingJobs)){
|
||||
LOGGER.debug("method=startJob||clusterPhyId={}||errMsg=no waiting job", clusterPhyId);
|
||||
return;
|
||||
}
|
||||
|
||||
//3、获取到 planTime 最近的一个任务，是否到达了任务执行时间
|
||||
Job job = waitingJobs.get(0);
|
||||
if(job.getPlanTime().after(new Date())){
|
||||
// 计划时间 > 当前时间，则直接返回
|
||||
return;
|
||||
}
|
||||
|
||||
// 计划时间 < 当前时间，则触发任务的执行
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
Result<Void> rv = jobHandler.process(job, JobActionEnum.START, Constant.DEFAULT_USER_NAME);
|
||||
if (rv.failed()) {
|
||||
LOGGER.error("method=startJob||clusterPhyId={}||errMsg=start job failed, {}!", clusterPhyId, rv.getMessage());
|
||||
return;
|
||||
}
|
||||
|
||||
// 调整任务状态为运行中
|
||||
this.setJobStartRunning(job);
|
||||
}catch (Exception e){
|
||||
LOGGER.error("method=startJob||clusterPhyId={}||errMsg=exception!", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void setJobStartRunning(Job job) {
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
Result<JobStatus> jobStatusRet = jobHandler.status(job);
|
||||
|
||||
if (jobStatusRet == null
|
||||
|| !jobStatusRet.successful()
|
||||
|| JobStatusEnum.isWaiting(jobStatusRet.getData().getStatus())){
|
||||
return;
|
||||
}
|
||||
job.setJobStatus(jobStatusRet.getData().getStatus());
|
||||
job.setStartTime(new Date());
|
||||
|
||||
job.setRunningStatus(JSON.toJSONString(jobStatusRet.getData()));
|
||||
jobDao.updateById(ConvertUtil.obj2Obj(job, JobPO.class));
|
||||
}
|
||||
|
||||
private void updateStatusJob(Long clusterPhyId){
|
||||
List<Job> runningJobs = this.listJobByClusterIdAndStatusSortByPlanTime(clusterPhyId, JobStatusEnum.RUNNING.getStatus());
|
||||
List<Job> waitingJobs = this.listJobByClusterIdAndStatusSortByPlanTime(clusterPhyId, JobStatusEnum.WAITING.getStatus());
|
||||
|
||||
List<Job> allJobs = new ArrayList<>();
|
||||
allJobs.addAll(runningJobs);
|
||||
allJobs.addAll(waitingJobs);
|
||||
|
||||
if(CollectionUtils.isEmpty(allJobs)){
|
||||
// 当前无任务，则直接返回
|
||||
return;
|
||||
}
|
||||
|
||||
for(Job job : allJobs){
|
||||
JobHandler jobHandler = getJobHandlerByType(job.getJobType());
|
||||
Result<JobStatus> jobStatusRet = jobHandler.status(job);
|
||||
|
||||
if(null != jobStatusRet && jobStatusRet.successful()) {
|
||||
job.setJobStatus(jobStatusRet.getData().getStatus());
|
||||
job.setRunningStatus(JSON.toJSONString(jobStatusRet.getData()));
|
||||
jobDao.updateById(ConvertUtil.obj2Obj(job, JobPO.class));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Job> listAllJobByClusterId(Long clusterPhyId){
|
||||
LambdaQueryWrapper<JobPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(JobPO::getClusterId, clusterPhyId);
|
||||
|
||||
return ConvertUtil.list2List(jobDao.selectList(queryWrapper), Job.class);
|
||||
}
|
||||
|
||||
private List<Job> listJobByClusterIdAndStatusSortByPlanTime(Long clusterPhyId, int status){
|
||||
LambdaQueryWrapper<JobPO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(JobPO::getClusterId, clusterPhyId);
|
||||
queryWrapper.eq(JobPO::getJobStatus, status);
|
||||
queryWrapper.orderByAsc(JobPO::getPlanTime);
|
||||
|
||||
return ConvertUtil.list2List(jobDao.selectList(queryWrapper), Job.class);
|
||||
}
|
||||
|
||||
private JobHandler getJobHandlerByType(int type){
|
||||
JobHandleEnum taskHandleEnum = JobHandleEnum.valueOfType(type);
|
||||
JobHandler handler = (JobHandler) handleFactory.getByHandlerNamePer(taskHandleEnum.getMessage());
|
||||
|
||||
return handler;
|
||||
}
|
||||
|
||||
private boolean insert(Job task) {
|
||||
try {
|
||||
JobPO jobPO = ConvertUtil.obj2Obj(task, JobPO.class);
|
||||
if (jobDao.insert( jobPO ) > 0) {
|
||||
task.setId(jobPO.getId());
|
||||
return true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=insert||taskType={}||errMsg={}", task.getJobType(), e.getStackTrace(), e);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private Map<Integer, String> getBrokerIdHostMap(Map<Integer, String> brokerIdHostMap, List<Integer> brokerIds){
|
||||
Map<Integer, String> ret = new HashMap<>();
|
||||
|
||||
for(Integer brokerId : brokerIds){
|
||||
ret.put(brokerId, brokerIdHostMap.get(brokerId));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
private Map<Integer, String> genBrokerIdHostMap(Long clusterPhyId, Set<Integer> brokerIds){
|
||||
Map<Integer, String> brokerIdHostMap = new HashMap<>();
|
||||
for(Integer brokerId : brokerIds){
|
||||
Broker broker = brokerService.getBroker(clusterPhyId, brokerId);
|
||||
if(null != broker){
|
||||
brokerIdHostMap.put(brokerId, broker.getHost());
|
||||
}
|
||||
}
|
||||
|
||||
return brokerIdHostMap;
|
||||
}
|
||||
|
||||
private Map<Integer, Tuple<List<Integer>/*source*/, List<Integer>/*des*/>> genBrokerSourceDesInfo(
|
||||
Map<Integer, Tuple<List<Integer>, List<Integer>>> brokerSourceDesMap,
|
||||
List<Integer> desBrokerIds, List<Integer> sourceBrokerIds){
|
||||
|
||||
for(Integer desBrokerId : desBrokerIds){
|
||||
brokerSourceDesMap.putIfAbsent(desBrokerId, new Tuple<>(new ArrayList<>(), new ArrayList<>()));
|
||||
brokerSourceDesMap.get(desBrokerId).getV1().addAll(sourceBrokerIds);
|
||||
}
|
||||
|
||||
for(Integer sourceBrokerId : sourceBrokerIds){
|
||||
brokerSourceDesMap.putIfAbsent(sourceBrokerId, new Tuple<>(new ArrayList<>(), new ArrayList<>()));
|
||||
brokerSourceDesMap.get(sourceBrokerId).getV2().addAll(desBrokerIds);
|
||||
}
|
||||
|
||||
return brokerSourceDesMap;
|
||||
}
|
||||
|
||||
private List<JobOverViewVO> jobPOS2JobVOS(List<JobPO> jobPOS){
|
||||
if(CollectionUtils.isEmpty(jobPOS)){return new ArrayList<>();}
|
||||
|
||||
return jobPOS.stream().map(j -> jobPO2JobVO(j)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private JobOverViewVO jobPO2JobVO(JobPO jobPO){
|
||||
JobOverViewVO jobOverViewVO = ConvertUtil.obj2Obj(jobPO, JobOverViewVO.class);
|
||||
JobStatus jobStatus = JSON.parseObject(jobPO.getRunningStatus(), JobStatus.class);
|
||||
if(null != jobStatus){
|
||||
jobOverViewVO.setDoing(jobStatus.getDoing());
|
||||
jobOverViewVO.setSuccess(jobStatus.getSuccess());
|
||||
jobOverViewVO.setFail(jobStatus.getFailed());
|
||||
jobOverViewVO.setTotal(jobStatus.getTotal());
|
||||
jobOverViewVO.setWaiting(jobStatus.getWaiting());
|
||||
}
|
||||
|
||||
return jobOverViewVO;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.kafkacontroller;
|
||||
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.kafkacontrollr.KafkaControllerPO;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface KafkaControllerService {
|
||||
Result<KafkaController> getControllerFromKafka(ClusterPhy clusterPhy);
|
||||
|
||||
int insertAndIgnoreDuplicateException(KafkaController kafkaController);
|
||||
|
||||
int setNoKafkaController(Long clusterPhyId, Long triggerTime);
|
||||
|
||||
KafkaController getKafkaControllerFromDB(Long clusterPhyId);
|
||||
|
||||
Map<Long, KafkaController> getKafkaControllersFromDB(List<Long> clusterPhyIdList, boolean notIncludeNotAlive);
|
||||
|
||||
IPage<KafkaControllerPO> pagingControllerHistories(Long clusterPhyId, Integer pageNo, Integer pageSize, String brokerHostSearchKeyword);
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.IPage;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkacontroller.KafkaController;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.kafkacontrollr.KafkaControllerPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterRunStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.kafkacontroller.KafkaControllerService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.kafkacontroller.KafkaControllerDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
@Service
|
||||
public class KafkaControllerServiceImpl implements KafkaControllerService {
|
||||
private static final ILog log = LogFactory.getLog(KafkaControllerServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaControllerDAO kafkaControllerDAO;
|
||||
|
||||
@Override
|
||||
public Result<KafkaController> getControllerFromKafka(ClusterPhy clusterPhy) {
|
||||
if (clusterPhy.getRunState().equals(ClusterRunStateEnum.RUN_ZK.getRunState())) {
|
||||
return this.getControllerFromZKClient(clusterPhy);
|
||||
}
|
||||
|
||||
return this.getControllerFromAdminClient(clusterPhy);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int insertAndIgnoreDuplicateException(KafkaController kafkaController) {
|
||||
try {
|
||||
Broker broker = brokerService.getBroker(kafkaController.getClusterPhyId(), kafkaController.getBrokerId());
|
||||
|
||||
KafkaControllerPO kafkaControllerPO = new KafkaControllerPO();
|
||||
kafkaControllerPO.setClusterPhyId(kafkaController.getClusterPhyId());
|
||||
kafkaControllerPO.setBrokerId(kafkaController.getBrokerId());
|
||||
kafkaControllerPO.setTimestamp(kafkaController.getTimestamp());
|
||||
kafkaControllerPO.setBrokerHost(broker != null? broker.getHost(): "");
|
||||
kafkaControllerPO.setBrokerRack(broker != null? broker.getRack(): "");
|
||||
kafkaControllerDAO.insert(kafkaControllerPO);
|
||||
} catch (DuplicateKeyException dke) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
// 删除错误的时间数据
|
||||
LambdaQueryWrapper<KafkaControllerPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaControllerPO::getClusterPhyId, kafkaController.getClusterPhyId());
|
||||
lambdaQueryWrapper.eq(KafkaControllerPO::getBrokerId, Constant.INVALID_CODE);
|
||||
lambdaQueryWrapper.gt(KafkaControllerPO::getTimestamp, kafkaController.getTimestamp());
|
||||
kafkaControllerDAO.delete(lambdaQueryWrapper);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int setNoKafkaController(Long clusterPhyId, Long triggerTime) {
|
||||
KafkaController kafkaController = this.getKafkaControllerFromDB(clusterPhyId);
|
||||
if (kafkaController == null) {
|
||||
// 已经被设置为no kafka-controller了
|
||||
return 0;
|
||||
}
|
||||
|
||||
KafkaController noKafkaController = new KafkaController();
|
||||
noKafkaController.setClusterPhyId(clusterPhyId);
|
||||
noKafkaController.setBrokerId(Constant.INVALID_CODE);
|
||||
|
||||
// 归一化到秒, 并且将去1秒，避免gc导致时间不对
|
||||
noKafkaController.setTimestamp(triggerTime);
|
||||
return this.insertAndIgnoreDuplicateException(noKafkaController);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaController getKafkaControllerFromDB(Long clusterPhyId) {
|
||||
Map<Long, KafkaController> controllerMap = this.getKafkaControllersFromDB(Arrays.asList(clusterPhyId), true);
|
||||
|
||||
return controllerMap.get(clusterPhyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<Long, KafkaController> getKafkaControllersFromDB(List<Long> clusterPhyIdList, boolean notIncludeNotAlive) {
|
||||
List<KafkaControllerPO> poList = kafkaControllerDAO.listAllLatest();
|
||||
|
||||
Map<Long, KafkaController> controllerMap = new HashMap<>();
|
||||
for (KafkaControllerPO po: poList) {
|
||||
if ((po.getBrokerId().equals(Constant.INVALID_CODE) && notIncludeNotAlive)
|
||||
|| !clusterPhyIdList.contains(po.getClusterPhyId())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
KafkaController kafkaController = new KafkaController();
|
||||
kafkaController.setClusterPhyId(po.getClusterPhyId());
|
||||
kafkaController.setBrokerId(po.getBrokerId());
|
||||
kafkaController.setTimestamp(po.getTimestamp());
|
||||
controllerMap.put(po.getClusterPhyId(), kafkaController);
|
||||
}
|
||||
|
||||
return controllerMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IPage<KafkaControllerPO> pagingControllerHistories(Long clusterPhyId, Integer pageNo, Integer pageSize, String brokerHostSearchKeyword) {
|
||||
LambdaQueryWrapper<KafkaControllerPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaControllerPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.like(!ValidateUtils.isBlank(brokerHostSearchKeyword), KafkaControllerPO::getBrokerHost, brokerHostSearchKeyword);
|
||||
lambdaQueryWrapper.orderByDesc(KafkaControllerPO::getTimestamp);
|
||||
|
||||
return kafkaControllerDAO.selectPage(new Page<>(pageNo, pageSize), lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private Result<KafkaController> getControllerFromAdminClient(ClusterPhy clusterPhy) {
|
||||
try {
|
||||
AdminClient adminClient = null;
|
||||
try {
|
||||
adminClient = kafkaAdminClient.getClient(clusterPhy.getId());
|
||||
} catch (Exception e) {
|
||||
log.error("class=KafkaControllerServiceImpl||method=getControllerFromAdminClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
// 集群已经加载进来，但是创建admin-client失败，则设置无controller
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
DescribeClusterResult describeClusterResult = adminClient.describeCluster(new DescribeClusterOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
|
||||
Node controllerNode = describeClusterResult.controller().get();
|
||||
if (controllerNode == null) {
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
return Result.buildSuc(new KafkaController(
|
||||
clusterPhy.getId(),
|
||||
controllerNode.id(),
|
||||
System.currentTimeMillis()
|
||||
));
|
||||
} catch (Exception e) {
|
||||
log.error("class=KafkaControllerServiceImpl||method=getControllerFromAdminClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<KafkaController> getControllerFromZKClient(ClusterPhy clusterPhy) {
|
||||
try {
|
||||
return Result.buildSuc(kafkaZKDAO.getKafkaController(clusterPhy.getId(), false));
|
||||
} catch (Exception e) {
|
||||
log.error("class=KafkaControllerServiceImpl||method=getControllerFromZKClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.kafkauser;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkauser.KafkaUser;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.kafkauser.KafkaUserParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.kafkauser.KafkaUserReplaceParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaUserPO;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
public interface KafkaUserService {
|
||||
/**************************************************** operate DB and Kafka Method ****************************************************/
|
||||
|
||||
/**
|
||||
* 新增KafkaUser
|
||||
*/
|
||||
Result<Void> createKafkaUser(KafkaUserReplaceParam param, String operator);
|
||||
|
||||
/**
|
||||
* 删除KafkaUser
|
||||
*/
|
||||
Result<Void> deleteKafkaUser(KafkaUserParam param, String operator);
|
||||
|
||||
/**
|
||||
* 修改KafkaUser
|
||||
*/
|
||||
Result<Void> modifyKafkaUser(KafkaUserReplaceParam param, String operator);
|
||||
|
||||
/**
|
||||
* 查询KafkaUser
|
||||
*/
|
||||
Result<List<KafkaUser>> getKafkaUserFromKafka(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 查询KafkaUser
|
||||
*/
|
||||
Result<KafkaUser> getKafkaUserFromKafka(Long clusterPhyId, String kafkaUser);
|
||||
|
||||
Result<Properties> generateScramCredential(Long clusterPhyId, String token);
|
||||
|
||||
boolean isTokenEqual2CredentialProps(Long clusterPhyId, Properties credentialProps, String token);
|
||||
|
||||
/**************************************************** operate DB-Method ****************************************************/
|
||||
|
||||
void batchReplaceKafkaUserInDB(Long clusterPhyId, List<String> kafkaUserList);
|
||||
|
||||
PaginationResult<KafkaUserPO> pagingKafkaUserFromDB(Long clusterPhyId, PaginationBaseDTO dto);
|
||||
|
||||
List<KafkaUserPO> getKafkaUserByClusterIdFromDB(Long clusterPhyId, String searchKafkaUserName);
|
||||
|
||||
List<KafkaUserPO> getKafkaUserFromDB(Long clusterPhyId);
|
||||
|
||||
KafkaUserPO getKafkaUserFromDB(Long clusterPhyId, String kafkaUser);
|
||||
|
||||
Integer countKafkaUserFromDB(Long clusterPhyId);
|
||||
|
||||
boolean checkExistKafkaUserFromDB(Long clusterPhyId, String kafkaUser);
|
||||
}
|
||||
@@ -0,0 +1,601 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.kafkauser.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.didiglobal.logi.security.util.PWEncryptUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.pagination.PaginationBaseDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.kafkauser.KafkaUser;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.kafkauser.KafkaUserParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.kafkauser.KafkaUserReplaceParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.KafkaUserPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.kafkauser.KafkaUserService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.KafkaUserDAO;
|
||||
import kafka.admin.ConfigCommand;
|
||||
import kafka.server.ConfigType;
|
||||
import kafka.zk.*;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.errors.UnsupportedVersionException;
|
||||
import org.apache.kafka.common.security.scram.ScramCredential;
|
||||
import org.apache.kafka.common.security.scram.internals.ScramCredentialUtils;
|
||||
import org.apache.kafka.common.security.scram.internals.ScramFormatter;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import scala.jdk.javaapi.CollectionConverters;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
|
||||
|
||||
@Service
|
||||
public class KafkaUserServiceImpl extends BaseVersionControlService implements KafkaUserService {
|
||||
private static final ILog log = LogFactory.getLog(KafkaUserServiceImpl.class);
|
||||
|
||||
private static final String KAFKA_USER_REPLACE = "replaceKafkaUser";
|
||||
private static final String KAFKA_USER_DELETE = "deleteKafkaUser";
|
||||
private static final String KAFKA_USER_GET = "getKafkaUser";
|
||||
|
||||
@Autowired
|
||||
private KafkaUserDAO kafkaUserDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.SERVICE_OP_KAFKA_USER;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(KAFKA_USER_REPLACE, V_0_10_1_0, V_2_7_2, "replaceKafkaUserByZKClient", this::replaceKafkaUserByZKClient);
|
||||
registerVCHandler(KAFKA_USER_REPLACE, V_2_7_2, V_MAX, "replaceKafkaUserByKafkaClient", this::replaceKafkaUserByKafkaClient);
|
||||
|
||||
registerVCHandler(KAFKA_USER_DELETE, V_0_10_1_0, V_2_7_2, "deleteKafkaUserByZKClient", this::deleteKafkaUserByZKClient);
|
||||
registerVCHandler(KAFKA_USER_DELETE, V_2_7_2, V_MAX, "deleteKafkaUserByKafkaClient", this::deleteKafkaUserByKafkaClient);
|
||||
|
||||
registerVCHandler(KAFKA_USER_GET, V_0_10_1_0, V_2_7_2, "getKafkaUserByZKClient", this::getKafkaUserByZKClient);
|
||||
registerVCHandler(KAFKA_USER_GET, V_2_7_2, V_MAX, "getKafkaUserByKafkaClient", this::getKafkaUserByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> createKafkaUser(KafkaUserReplaceParam param, String operator) {
|
||||
if (this.checkExistKafkaUserFromDB(param.getClusterPhyId(), param.getKafkaUserName())) {
|
||||
// kafka-user已存在，则返回错误
|
||||
return Result.buildFromRSAndMsg(ResultStatus.DUPLICATION, MsgConstant.getKafkaUserDuplicate(param.getClusterPhyId(), param.getKafkaUserName()));
|
||||
}
|
||||
|
||||
Result<Void> rv = null;
|
||||
|
||||
// 修改Kafka
|
||||
try {
|
||||
rv = (Result<Void>) versionControlService.doHandler(getVersionItemType(), getMethodName(param.getClusterPhyId(), KAFKA_USER_REPLACE), param);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 写DB
|
||||
rv = this.replaceKafkaUserInDB(param.getClusterPhyId(), param.getKafkaUserName(), param.getKafkaUserToken());
|
||||
if (rv.failed()) {
|
||||
// rv中提示创建成功但是写DB失败
|
||||
rv = Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, "创建KafkaUser成功，但写入DB失败，错误信息: " + rv.getMessage());
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.ADD.getDesc(),
|
||||
ModuleEnum.KAFKA_USER.getDesc(),
|
||||
MsgConstant.getKafkaUserBizStr(param.getClusterPhyId(), param.getKafkaUserName()),
|
||||
param.toString()
|
||||
));
|
||||
|
||||
// 返回结果
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> deleteKafkaUser(KafkaUserParam param, String operator) {
|
||||
// 查询DB判断是否存在
|
||||
if (!this.checkExistKafkaUserFromDB(param.getClusterPhyId(), param.getKafkaUserName())) {
|
||||
// kafka-user不存在，则返回错误
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getKafkaUserNotExist(param.getClusterPhyId(), param.getKafkaUserName()));
|
||||
}
|
||||
|
||||
Result<Void> rv = null;
|
||||
|
||||
// 修改Kafka
|
||||
try {
|
||||
rv = (Result<Void>) versionControlService.doHandler(getVersionItemType(), getMethodName(param.getClusterPhyId(), KAFKA_USER_DELETE), param);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 写DB
|
||||
rv = this.deleteKafkaUserInDB(param.getClusterPhyId(), param.getKafkaUserName());
|
||||
if (rv.failed()) {
|
||||
// rv中提示删除成功但是写DB失败
|
||||
rv = Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, "删除Kafka-User成功，但删除DB数据失败，错误信息: " + rv.getMessage());
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.DELETE.getDesc(),
|
||||
ModuleEnum.KAFKA_USER.getDesc(),
|
||||
MsgConstant.getKafkaUserBizStr(param.getClusterPhyId(), param.getKafkaUserName()),
|
||||
param.toString()
|
||||
));
|
||||
|
||||
// 返回结果
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> modifyKafkaUser(KafkaUserReplaceParam param, String operator) {
|
||||
// 查询DB判断是否存在
|
||||
if (!this.checkExistKafkaUserFromDB(param.getClusterPhyId(), param.getKafkaUserName())) {
|
||||
// kafka-user不存在，则返回错误
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getKafkaUserNotExist(param.getClusterPhyId(), param.getKafkaUserName()));
|
||||
}
|
||||
|
||||
Result<Void> rv = null;
|
||||
|
||||
// 修改Kafka
|
||||
try {
|
||||
rv = (Result<Void>) versionControlService.doHandler(getVersionItemType(), getMethodName(param.getClusterPhyId(), KAFKA_USER_REPLACE), param);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 写DB
|
||||
rv = this.replaceKafkaUserInDB(param.getClusterPhyId(), param.getKafkaUserName(), param.getKafkaUserToken());
|
||||
if (rv.failed()) {
|
||||
// rv中提示创建成功但是写DB失败
|
||||
rv = Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, "修改KafkaUser成功，但写入DB失败，错误信息: " + rv.getMessage());
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.KAFKA_USER.getDesc(),
|
||||
MsgConstant.getKafkaUserBizStr(param.getClusterPhyId(), param.getKafkaUserName()),
|
||||
param.toString()
|
||||
));
|
||||
|
||||
// 返回结果
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<KafkaUser>> getKafkaUserFromKafka(Long clusterPhyId) {
|
||||
try {
|
||||
return (Result<List<KafkaUser>>) versionControlService.doHandler(
|
||||
getVersionItemType(),
|
||||
getMethodName(clusterPhyId, KAFKA_USER_GET),
|
||||
new KafkaUserParam(clusterPhyId, null)
|
||||
);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<KafkaUser> getKafkaUserFromKafka(Long clusterPhyId, String kafkaUser) {
|
||||
try {
|
||||
Result<List<KafkaUser>> listResult = (Result<List<KafkaUser>>) versionControlService.doHandler(
|
||||
getVersionItemType(),
|
||||
getMethodName(clusterPhyId, KAFKA_USER_GET),
|
||||
new KafkaUserParam(clusterPhyId, kafkaUser)
|
||||
);
|
||||
|
||||
if (listResult.failed()) {
|
||||
return Result.buildFromIgnoreData(listResult);
|
||||
}
|
||||
|
||||
if (ValidateUtils.isEmptyList(listResult.getData())) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getKafkaUserNotExist(clusterPhyId, kafkaUser));
|
||||
}
|
||||
|
||||
return Result.buildSuc(listResult.getData().get(0));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Properties> generateScramCredential(Long clusterPhyId, String token) {
|
||||
try {
|
||||
Properties props = new Properties();
|
||||
|
||||
props.put(ScramMechanism.SCRAM_SHA_256.mechanismName(),
|
||||
ScramCredentialUtils.credentialToString(new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.SCRAM_SHA_256).generateCredential(token, ConfigCommand.DefaultScramIterations()))
|
||||
);
|
||||
|
||||
props.put(ScramMechanism.SCRAM_SHA_512.mechanismName(),
|
||||
ScramCredentialUtils.credentialToString(new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.SCRAM_SHA_512).generateCredential(token, ConfigCommand.DefaultScramIterations()))
|
||||
);
|
||||
|
||||
return Result.buildSuc(props);
|
||||
} catch (Exception e) {
|
||||
log.error("method=generateScramCredential||clusterPhyId={}||token={}||errMsg=exception", clusterPhyId, token, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isTokenEqual2CredentialProps(Long clusterPhyId, Properties credentialProps, String token) {
|
||||
try {
|
||||
boolean existScram = false;
|
||||
boolean existNotEqual = false;
|
||||
for (ScramMechanism scramMechanism: ScramMechanism.values()) {
|
||||
String value = credentialProps.getProperty(scramMechanism.mechanismName());
|
||||
if (ValidateUtils.isBlank(value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
existScram = true;
|
||||
ScramCredential rawScramCredential = ScramCredentialUtils.credentialFromString(value);
|
||||
|
||||
ScramFormatter scramFormatter = ScramMechanism.SCRAM_SHA_256.equals(scramMechanism)?
|
||||
new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.SCRAM_SHA_256):
|
||||
new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.SCRAM_SHA_512);
|
||||
|
||||
ScramCredential tokenScramCredential = scramFormatter.generateCredential(
|
||||
rawScramCredential.salt(),
|
||||
scramFormatter.saltedPassword(token, rawScramCredential.salt(), rawScramCredential.iterations()),
|
||||
rawScramCredential.iterations()
|
||||
);
|
||||
|
||||
if (!ScramCredentialUtils.credentialToString(rawScramCredential).equals(ScramCredentialUtils.credentialToString(tokenScramCredential))) {
|
||||
// 存在不相等的情况
|
||||
existNotEqual = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 存在，并且没有不相等的情况，这返回true
|
||||
return existScram && !existNotEqual;
|
||||
} catch (Exception e) {
|
||||
log.error("method=isTokenEqual2CredentialProps||clusterPhyId={}||credentialProps={}||token={}||errMsg=exception", clusterPhyId, credentialProps, token, e);
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** operate DB-Method ****************************************************/
|
||||
|
||||
@Override
|
||||
public void batchReplaceKafkaUserInDB(Long clusterPhyId, List<String> kafkaUserList) {
|
||||
Map<String, KafkaUserPO> poMap = this.getKafkaUserFromDB(clusterPhyId).stream().collect(Collectors.toMap(KafkaUserPO::getName, Function.identity()));
|
||||
|
||||
for (String kafkaUser: kafkaUserList) {
|
||||
KafkaUserPO kafkaUserPO = poMap.remove(kafkaUser);
|
||||
if (kafkaUserPO != null) {
|
||||
// 已存在，则略过
|
||||
continue;
|
||||
}
|
||||
|
||||
// 不存时则插入
|
||||
try {
|
||||
kafkaUserPO = new KafkaUserPO();
|
||||
kafkaUserPO.setClusterPhyId(clusterPhyId);
|
||||
kafkaUserPO.setName(kafkaUser);
|
||||
kafkaUserDAO.insert(kafkaUserPO);
|
||||
} catch (DuplicateKeyException dke) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
// 删除不存在的
|
||||
for (KafkaUserPO kafkaUserPO: poMap.values()) {
|
||||
kafkaUserDAO.deleteById(kafkaUserPO.getClusterPhyId());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<KafkaUserPO> pagingKafkaUserFromDB(Long clusterPhyId, PaginationBaseDTO dto) {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.like(!ValidateUtils.isBlank(dto.getSearchKeywords()), KafkaUserPO::getName, dto.getSearchKeywords());
|
||||
lambdaQueryWrapper.orderByDesc(KafkaUserPO::getCreateTime);
|
||||
|
||||
Page<KafkaUserPO> poPage = kafkaUserDAO.selectPage(new Page<>(dto.getPageNo(), dto.getPageSize()), lambdaQueryWrapper);
|
||||
|
||||
return PaginationResult.buildSuc(poPage.getRecords(), poPage);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KafkaUserPO> getKafkaUserByClusterIdFromDB(Long clusterPhyId, String searchKafkaUserName) {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.like(!ValidateUtils.isBlank(searchKafkaUserName), KafkaUserPO::getName, searchKafkaUserName);
|
||||
|
||||
return kafkaUserDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KafkaUserPO> getKafkaUserFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return kafkaUserDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KafkaUserPO getKafkaUserFromDB(Long clusterPhyId, String kafkaUser) {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getName, kafkaUser);
|
||||
|
||||
return kafkaUserDAO.selectOne(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countKafkaUserFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return kafkaUserDAO.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean checkExistKafkaUserFromDB(Long clusterPhyId, String kafkaUser) {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getName, kafkaUser);
|
||||
|
||||
return kafkaUserDAO.selectCount(lambdaQueryWrapper) > 0;
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private Result<Void> replaceKafkaUserInDB(Long clusterPhyId, String kafkaUser, String rowToken) {
|
||||
try {
|
||||
boolean exist = true;
|
||||
|
||||
KafkaUserPO kafkaUserPO = this.getKafkaUserFromDB(clusterPhyId, kafkaUser);
|
||||
if (kafkaUserPO == null) {
|
||||
kafkaUserPO = new KafkaUserPO();
|
||||
exist = false;
|
||||
}
|
||||
|
||||
kafkaUserPO.setClusterPhyId(clusterPhyId);
|
||||
kafkaUserPO.setName(kafkaUser);
|
||||
kafkaUserPO.setToken(PWEncryptUtil.encode(rowToken));
|
||||
|
||||
if (!exist) {
|
||||
kafkaUserDAO.insert(kafkaUserPO);
|
||||
} else {
|
||||
kafkaUserDAO.updateById(kafkaUserPO);
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=insertKafkaUserToDB||clusterPhyId={}||kafkaUser={}||errMsg=exception.", clusterPhyId, kafkaUser, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> deleteKafkaUserInDB(Long clusterPhyId, String kafkaUser) {
|
||||
try {
|
||||
LambdaQueryWrapper<KafkaUserPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(KafkaUserPO::getName, kafkaUser);
|
||||
|
||||
kafkaUserDAO.delete(lambdaQueryWrapper);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=deleteKafkaUserInDB||clusterPhyId={}||kafkaUser={}||errMsg=exception.", clusterPhyId, kafkaUser, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> replaceKafkaUserByZKClient(VersionItemParam itemParam) {
|
||||
KafkaUserReplaceParam param = (KafkaUserReplaceParam) itemParam;
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(param.getClusterPhyId());
|
||||
Properties properties = adminZkClient.fetchEntityConfig(ConfigType.User(), param.getKafkaUserName());
|
||||
|
||||
// 没有开放的接口，因此ZK的需要自己进行实现
|
||||
properties.put(ScramMechanism.SCRAM_SHA_256.mechanismName(),
|
||||
ScramCredentialUtils.credentialToString(new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.SCRAM_SHA_256).generateCredential(param.getKafkaUserToken(), ConfigCommand.DefaultScramIterations()))
|
||||
);
|
||||
properties.put(ScramMechanism.SCRAM_SHA_512.mechanismName(),
|
||||
ScramCredentialUtils.credentialToString(new ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramMechanism.SCRAM_SHA_512).generateCredential(param.getKafkaUserToken(), ConfigCommand.DefaultScramIterations()))
|
||||
);
|
||||
|
||||
adminZkClient.changeConfigs(ConfigType.User(), param.getKafkaUserName(), properties);
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=replaceKafkaUserByZKClient||parma={}||errMsg={}", itemParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> replaceKafkaUserByKafkaClient(VersionItemParam itemParam) {
|
||||
KafkaUserReplaceParam param = (KafkaUserReplaceParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
AlterUserScramCredentialsResult alterUserScramCredentialsResult = adminClient.alterUserScramCredentials(
|
||||
Arrays.asList(
|
||||
// scram-256
|
||||
new UserScramCredentialUpsertion(
|
||||
param.getKafkaUserName(),
|
||||
new ScramCredentialInfo(ScramMechanism.SCRAM_SHA_256, ConfigCommand.DefaultScramIterations()),
|
||||
param.getKafkaUserToken().getBytes(StandardCharsets.UTF_8),
|
||||
ScramFormatter.secureRandomBytes(new SecureRandom())),
|
||||
|
||||
// scram-512
|
||||
new UserScramCredentialUpsertion(
|
||||
param.getKafkaUserName(),
|
||||
new ScramCredentialInfo(ScramMechanism.SCRAM_SHA_512, ConfigCommand.DefaultScramIterations()),
|
||||
param.getKafkaUserToken().getBytes(StandardCharsets.UTF_8),
|
||||
ScramFormatter.secureRandomBytes(new SecureRandom()))
|
||||
),
|
||||
new AlterUserScramCredentialsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
alterUserScramCredentialsResult.all().get();
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=replaceKafkaUserByKafkaClient||parma={}||errMsg={}", itemParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> deleteKafkaUserByZKClient(VersionItemParam itemParam) {
|
||||
KafkaUserParam param = (KafkaUserParam) itemParam;
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(param.getClusterPhyId());
|
||||
|
||||
// 删除kafka-user
|
||||
kafkaZkClient.deletePath(ConfigEntityZNode.path(ConfigType.User(), param.getKafkaUserName()), ZkVersion.MatchAnyVersion(), false);
|
||||
|
||||
kafkaZkClient.createConfigChangeNotification(ConfigType.User() + "/" + param.getKafkaUserName());
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=deleteKafkaUserByZKClient||parma={}||errMsg={}", itemParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> deleteKafkaUserByKafkaClient(VersionItemParam itemParam) {
|
||||
KafkaUserParam param = (KafkaUserParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
AlterUserScramCredentialsResult alterUserScramCredentialsResult = adminClient.alterUserScramCredentials(
|
||||
Arrays.asList(
|
||||
// scram-256
|
||||
new UserScramCredentialDeletion(
|
||||
param.getKafkaUserName(),
|
||||
ScramMechanism.SCRAM_SHA_256),
|
||||
|
||||
// scram-512
|
||||
new UserScramCredentialDeletion(
|
||||
param.getKafkaUserName(),
|
||||
ScramMechanism.SCRAM_SHA_512)
|
||||
),
|
||||
new AlterUserScramCredentialsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
alterUserScramCredentialsResult.all().get();
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=deleteKafkaUserByKafkaClient||parma={}||errMsg={}", itemParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<KafkaUser>> getKafkaUserByZKClient(VersionItemParam itemParam) {
|
||||
KafkaUserParam param = (KafkaUserParam) itemParam;
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(param.getClusterPhyId());
|
||||
|
||||
List<String> kafkaUserNameList = null;
|
||||
if (ValidateUtils.isBlank(param.getKafkaUserName())) {
|
||||
kafkaUserNameList = CollectionConverters.asJava(kafkaZkClient.getChildren(ConfigEntityTypeZNode.path(ConfigType.User())));
|
||||
} else {
|
||||
kafkaUserNameList = Arrays.asList(param.getKafkaUserName());
|
||||
}
|
||||
|
||||
List<KafkaUser> kafkaUserList = new ArrayList<>();
|
||||
for (String kafkaUser: kafkaUserNameList) {
|
||||
Properties properties = kafkaZkClient.getEntityConfigs(ConfigType.User(), param.getKafkaUserName());
|
||||
|
||||
kafkaUserList.add(new KafkaUser(param.getClusterPhyId(), kafkaUser, null, properties));
|
||||
}
|
||||
|
||||
return Result.buildSuc(kafkaUserList);
|
||||
} catch (Exception e) {
|
||||
log.error("method=getKafkaUserByZKClient||clusterPhyId={}||kafkaUser={}||errMsg=exception", param.getClusterPhyId(), param.getKafkaUserName(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<KafkaUser>> getKafkaUserByKafkaClient(VersionItemParam itemParam) {
|
||||
KafkaUserParam param = (KafkaUserParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
// 查询集群kafka-user
|
||||
DescribeUserScramCredentialsResult describeUserScramCredentialsResult = null;
|
||||
if (ValidateUtils.isBlank(param.getKafkaUserName())) {
|
||||
describeUserScramCredentialsResult = adminClient.describeUserScramCredentials();
|
||||
} else {
|
||||
describeUserScramCredentialsResult = adminClient.describeUserScramCredentials(
|
||||
Arrays.asList(param.getKafkaUserName()),
|
||||
new DescribeUserScramCredentialsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
}
|
||||
|
||||
Map<String, UserScramCredentialsDescription> descriptionMap = describeUserScramCredentialsResult.all().get();
|
||||
|
||||
List<KafkaUser> kafkaUserList = new ArrayList<>();
|
||||
for (Map.Entry<String, UserScramCredentialsDescription> entry: descriptionMap.entrySet()) {
|
||||
kafkaUserList.add(new KafkaUser(param.getClusterPhyId(), entry.getKey(), null, new Properties()));
|
||||
}
|
||||
|
||||
return Result.buildSuc(kafkaUserList);
|
||||
} catch (Exception e) {
|
||||
log.error("method=getKafkaUserByKafkaClient||clusterPhyId={}||kafkaUser={}||errMsg=exception", param.getClusterPhyId(), param.getKafkaUserName(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.km;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface KmNodeService {
|
||||
|
||||
/**
|
||||
* 获取 km 集群所有部署的 host 节点
|
||||
* @return
|
||||
*/
|
||||
List<String> listKmHosts();
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.km.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.km.KmNodePO;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.NetUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.km.KmNodeService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.km.KmNodeDAO;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class KmNodeServiceImpl implements KmNodeService {
|
||||
private static final ILog LOGGER = LogFactory.getLog(KmNodeServiceImpl.class);
|
||||
|
||||
public static final long CLEAR_INTERVAL = 3 * 10 * 1000L;
|
||||
|
||||
@Autowired
|
||||
private KmNodeDAO kmNodeDAO;
|
||||
|
||||
@Value("${spring.application.name}")
|
||||
private String appName;
|
||||
|
||||
@PostConstruct
|
||||
public void init(){
|
||||
beat();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listKmHosts(){
|
||||
List<KmNodePO> kmNodePOS = listByAppName(appName);
|
||||
|
||||
if(CollectionUtils.isEmpty(kmNodePOS)){return new ArrayList<>();}
|
||||
|
||||
return kmNodePOS.stream().map(KmNodePO::getHostName).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Scheduled(cron="0/10 * * * * ?")
|
||||
public boolean beat() {
|
||||
clear();
|
||||
|
||||
try {
|
||||
String host = NetUtils.localHost();
|
||||
KmNodePO kmNodePO = selectByHostAndAppName(host, appName);
|
||||
if(null != kmNodePO){
|
||||
updateBeatTime(kmNodePO);
|
||||
}else {
|
||||
kmNodePO = new KmNodePO();
|
||||
kmNodePO.setBeatTime(new Date());
|
||||
kmNodePO.setHostName(host);
|
||||
kmNodePO.setIp(NetUtils.localIp());
|
||||
kmNodePO.setAppName(appName);
|
||||
|
||||
kmNodeDAO.insert(kmNodePO);
|
||||
}
|
||||
return true;
|
||||
}catch (Exception e){
|
||||
LOGGER.error("method=beat||msg=exception!", e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
private void clear(){
|
||||
long currentTime = System.currentTimeMillis();
|
||||
|
||||
List<KmNodePO> kmNodePOS = listByAppName(appName);
|
||||
if(CollectionUtils.isEmpty(kmNodePOS)){return;}
|
||||
|
||||
for (KmNodePO kmNodePO : kmNodePOS) {
|
||||
if (kmNodePO.getBeatTime().getTime() + CLEAR_INTERVAL < currentTime) {
|
||||
kmNodeDAO.deleteById(kmNodePO.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<KmNodePO> listByAppName(String appName){
|
||||
LambdaQueryWrapper<KmNodePO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KmNodePO::getAppName, appName);
|
||||
|
||||
return kmNodeDAO.selectList(queryWrapper);
|
||||
}
|
||||
|
||||
private KmNodePO selectByHostAndAppName(String hostName, String appName){
|
||||
LambdaQueryWrapper<KmNodePO> queryWrapper = new LambdaQueryWrapper<>();
|
||||
queryWrapper.eq(KmNodePO::getHostName, hostName);
|
||||
queryWrapper.eq(KmNodePO::getAppName, appName);
|
||||
|
||||
return kmNodeDAO.selectOne(queryWrapper);
|
||||
}
|
||||
|
||||
int updateBeatTime(KmNodePO kmNodePO){
|
||||
kmNodePO.setBeatTime(new Date());
|
||||
|
||||
return kmNodeDAO.updateById(kmNodePO);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.oprecord;
|
||||
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
|
||||
public interface OpLogWrapService {
|
||||
Integer saveOplogAndIgnoreException(OplogDTO oplogDTO);
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.oprecord.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.didiglobal.logi.security.service.OplogService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class OpLogWrapServiceImpl implements OpLogWrapService {
|
||||
private static final ILog log = LogFactory.getLog(OpLogWrapServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private OplogService oplogService;
|
||||
|
||||
@Override
|
||||
public Integer saveOplogAndIgnoreException(OplogDTO oplogDTO) {
|
||||
try {
|
||||
return oplogService.saveOplog(oplogDTO);
|
||||
} catch (Exception e) {
|
||||
log.error("method=saveOplogAndIgnoreException||oplogDTO={}||errMsg=exception.", oplogDTO, e);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.partition;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public interface PartitionMetricService {
|
||||
|
||||
/**
|
||||
* 从Kafka获取分区指标
|
||||
*/
|
||||
Result<List<PartitionMetrics>> collectPartitionsMetricsFromKafkaWithCache(Long clusterPhyId, String topicName, String metricName);
|
||||
|
||||
Result<List<PartitionMetrics>> collectPartitionsMetricsFromKafka(Long clusterPhyId, String topicName, String metricName);
|
||||
|
||||
Result<PartitionMetrics> collectPartitionMetricsFromKafka(Long clusterPhyId, String topicName, Integer partitionId, String metricName);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 从ES获取指标
|
||||
*/
|
||||
PartitionMetrics getLatestMetricsFromES(Long clusterPhyId, String topic, Integer brokerId, Integer partitionId, List<String> metricNameList);
|
||||
|
||||
Result<List<PartitionMetrics>> getLatestMetricsFromES(Long clusterPhyId, String topicName, List<String> metricNameList);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.partition;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.partition.PartitionPO;
|
||||
import org.apache.kafka.clients.admin.OffsetSpec;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public interface PartitionService {
|
||||
Result<Map<String, List<Partition>>> listPartitionsFromKafka(ClusterPhy clusterPhy);
|
||||
|
||||
List<Partition> listPartitionByCluster(Long clusterPhyId);
|
||||
List<PartitionPO> listPartitionPOByCluster(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* Topic下的分区列表
|
||||
*/
|
||||
List<Partition> listPartitionByTopic(Long clusterPhyId, String topicName);
|
||||
|
||||
|
||||
/**
|
||||
* Broker下的分区列表
|
||||
*/
|
||||
List<Partition> listPartitionByBroker(Long clusterPhyId, Integer brokerId);
|
||||
|
||||
/**
|
||||
* 获取具体分区信息
|
||||
*/
|
||||
Partition getPartitionByTopicAndPartitionId(Long clusterPhyId, String topicName, Integer partitionId);
|
||||
|
||||
|
||||
/**************************************************** 优先从缓存获取分区信息 ****************************************************/
|
||||
|
||||
List<Partition> listPartitionFromCacheFirst(Long clusterPhyId, String topicName);
|
||||
Partition getPartitionFromCacheFirst(Long clusterPhyId, String topicName, Integer partitionId);
|
||||
|
||||
/**
|
||||
* 获取集群下分区数
|
||||
*/
|
||||
Integer getPartitionSizeByClusterId(Long clusterPhyId);
|
||||
|
||||
Integer getLeaderPartitionSizeByClusterId(Long clusterPhyId);
|
||||
|
||||
Integer getNoLeaderPartitionSizeByClusterId(Long clusterPhyId);
|
||||
|
||||
Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp);
|
||||
|
||||
Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, Integer partitionId, OffsetSpec offsetSpec, Long timestamp);
|
||||
|
||||
int updatePartitions(Long clusterPhyId, String topicName, List<Partition> kafkaPartitionList, List<PartitionPO> dbPartitionList);
|
||||
|
||||
void deletePartitionsIfNotIn(Long clusterPhyId, Set<String> topicNameSet);
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.partition.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.TopicMetricParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.PartitionMetricPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BeanUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.cache.CollectMetricsLocalCache;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.PartitionMetricESDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||
import org.apache.kafka.clients.admin.OffsetSpec;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.management.InstanceNotFoundException;
|
||||
import javax.management.ObjectName;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.PartitionMetricVersionItems.*;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service
|
||||
public class PartitionMetricServiceImpl extends BaseMetricService implements PartitionMetricService {
|
||||
private static final ILog LOGGER = LogFactory.getLog(PartitionMetricServiceImpl.class);
|
||||
|
||||
public static final String PARTITION_METHOD_GET_METRIC_FROM_JMX = "getMetricFromJmx";
|
||||
public static final String PARTITION_METHOD_GET_OFFSET_RELEVANT_METRICS = "getOffsetRelevantMetrics";
|
||||
public static final String PARTITION_METHOD_GET_TOPIC_AVG_METRIC_MESSAGES = "getTopicAvgMetricFromJmx";
|
||||
|
||||
@Autowired
|
||||
private KafkaJMXClient kafkaJMXClient;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private PartitionMetricESDAO partitionMetricESDAO;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.METRIC_PARTITION;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<String> listMetricPOFields(){
|
||||
return BeanUtil.listBeanFields(PartitionMetricPO.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initRegisterVCHandler(){
|
||||
registerVCHandler( PARTITION_METHOD_GET_METRIC_FROM_JMX, this::getMetricFromJmx);
|
||||
registerVCHandler( PARTITION_METHOD_GET_OFFSET_RELEVANT_METRICS, this::getOffsetRelevantMetrics);
|
||||
registerVCHandler( PARTITION_METHOD_GET_TOPIC_AVG_METRIC_MESSAGES, this::getTopicAvgMetricFromJmx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<PartitionMetrics>> collectPartitionsMetricsFromKafkaWithCache(Long clusterPhyId, String topicName, String metricName) {
|
||||
List<PartitionMetrics> metricsList = CollectMetricsLocalCache.getPartitionMetricsList(clusterPhyId, topicName, metricName);
|
||||
if(null != metricsList) {
|
||||
return Result.buildSuc(metricsList);
|
||||
}
|
||||
|
||||
Result<List<PartitionMetrics>> metricsResult = this.collectPartitionsMetricsFromKafka(clusterPhyId, topicName, metricName);
|
||||
if(null == metricsResult || metricsResult.failed() || null == metricsResult.getData() || metricsResult.getData().isEmpty()) {
|
||||
return metricsResult;
|
||||
}
|
||||
|
||||
// 更新cache
|
||||
PartitionMetrics metrics = metricsResult.getData().get(0);
|
||||
metrics.getMetrics().entrySet().forEach(
|
||||
metricEntry -> CollectMetricsLocalCache.putPartitionMetricsList(
|
||||
clusterPhyId,
|
||||
metrics.getTopic(),
|
||||
metricEntry.getKey(),
|
||||
metricsResult.getData()
|
||||
)
|
||||
);
|
||||
|
||||
return metricsResult;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<PartitionMetrics> collectPartitionMetricsFromKafka(Long clusterPhyId, String topicName, Integer partitionId, String metricName){
|
||||
Result<List<PartitionMetrics>> metricsResult = this.collectPartitionsMetricsFromKafka(clusterPhyId, topicName, metricName);
|
||||
if (!metricsResult.hasData()) {
|
||||
return Result.buildFromIgnoreData(metricsResult);
|
||||
}
|
||||
|
||||
for (PartitionMetrics metrics: metricsResult.getData()) {
|
||||
if (metrics.getPartitionId().equals(partitionId)) {
|
||||
return Result.buildSuc(metrics);
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildFromRSAndMsg(KAFKA_OPERATE_FAILED, "Topic分区指标获取成功，但是该分区的指标不存在");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<PartitionMetrics>> collectPartitionsMetricsFromKafka(Long clusterPhyId, String topicName, String metricName) {
|
||||
try {
|
||||
TopicMetricParam param = new TopicMetricParam(clusterPhyId, topicName, metricName);
|
||||
|
||||
return (Result<List<PartitionMetrics>>) doVCHandler(clusterPhyId, metricName, param);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PartitionMetrics getLatestMetricsFromES(Long clusterPhyId, String topic, Integer brokerId, Integer partitionId, List<String> metricNames) {
|
||||
PartitionMetricPO po = partitionMetricESDAO.getPartitionLatestMetrics(clusterPhyId, topic, brokerId, partitionId, metricNames);
|
||||
|
||||
return ConvertUtil.obj2Obj(po, PartitionMetrics.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<PartitionMetrics>> getLatestMetricsFromES(Long clusterPhyId, String topicName, List<String> metricNames) {
|
||||
List<PartitionMetricPO> poList = partitionMetricESDAO.listPartitionLatestMetricsByTopic(clusterPhyId, topicName, metricNames);
|
||||
|
||||
return Result.buildSuc(ConvertUtil.list2List(poList, PartitionMetrics.class));
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private Result<List<PartitionMetrics>> getOffsetRelevantMetrics(VersionItemParam param) {
|
||||
TopicMetricParam metricParam = (TopicMetricParam) param;
|
||||
String metricName = metricParam.getMetric();
|
||||
String topicName = metricParam.getTopic();
|
||||
Long clusterPhyId = metricParam.getClusterId();
|
||||
|
||||
List<Partition> partitionList = partitionService.listPartitionFromCacheFirst(clusterPhyId, topicName);
|
||||
Map<Integer, Partition> partitionMap = partitionList.stream().collect(Collectors.toMap(Partition::getPartitionId, Function.identity()));
|
||||
|
||||
Map<Integer, PartitionMetrics> metricsMap = new HashMap<>();
|
||||
|
||||
// begin offset 指标
|
||||
Result<Map<TopicPartition, Long>> beginOffsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, OffsetSpec.earliest(), null);
|
||||
if (beginOffsetMapResult.hasData()) {
|
||||
for (Map.Entry<TopicPartition, Long> entry: beginOffsetMapResult.getData().entrySet()) {
|
||||
Partition partition = partitionMap.get(entry.getKey().partition());
|
||||
PartitionMetrics metrics = metricsMap.getOrDefault(
|
||||
entry.getKey().partition(),
|
||||
new PartitionMetrics(clusterPhyId, topicName, partition != null? partition.getLeaderBrokerId(): KafkaConstant.NO_LEADER, entry.getKey().partition())
|
||||
);
|
||||
|
||||
metrics.putMetric(PARTITION_METRIC_LOG_START_OFFSET, entry.getValue().floatValue());
|
||||
metricsMap.put(entry.getKey().partition(), metrics);
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn(
|
||||
"class=PartitionMetricServiceImpl||method=getOffsetRelevantMetrics||clusterPhyId={}||topicName={}||resultMsg={}||msg=get begin offset failed",
|
||||
clusterPhyId, topicName, beginOffsetMapResult.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
// end offset 指标
|
||||
Result<Map<TopicPartition, Long>> endOffsetMapResult = partitionService.getPartitionOffsetFromKafka(clusterPhyId, topicName, OffsetSpec.latest(), null);
|
||||
if (endOffsetMapResult.hasData()) {
|
||||
for (Map.Entry<TopicPartition, Long> entry: endOffsetMapResult.getData().entrySet()) {
|
||||
Partition partition = partitionMap.get(entry.getKey().partition());
|
||||
PartitionMetrics metrics = metricsMap.getOrDefault(
|
||||
entry.getKey().partition(),
|
||||
new PartitionMetrics(clusterPhyId, topicName, partition != null? partition.getLeaderBrokerId(): KafkaConstant.NO_LEADER, entry.getKey().partition())
|
||||
);
|
||||
|
||||
metrics.putMetric(PARTITION_METRIC_LOG_END_OFFSET, entry.getValue().floatValue());
|
||||
metricsMap.put(entry.getKey().partition(), metrics);
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn(
|
||||
"class=PartitionMetricServiceImpl||method=getOffsetRelevantMetrics||clusterPhyId={}||topicName={}||resultMsg={}||msg=get end offset failed",
|
||||
clusterPhyId, topicName, endOffsetMapResult.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
// messages 指标
|
||||
if (endOffsetMapResult.hasData() && beginOffsetMapResult.hasData()) {
|
||||
for (Map.Entry<TopicPartition, Long> entry: endOffsetMapResult.getData().entrySet()) {
|
||||
Long beginOffset = beginOffsetMapResult.getData().get(entry.getKey());
|
||||
if (beginOffset == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Partition partition = partitionMap.get(entry.getKey().partition());
|
||||
PartitionMetrics metrics = metricsMap.getOrDefault(
|
||||
entry.getKey().partition(),
|
||||
new PartitionMetrics(clusterPhyId, topicName, partition != null? partition.getLeaderBrokerId(): KafkaConstant.NO_LEADER, entry.getKey().partition())
|
||||
);
|
||||
|
||||
metrics.putMetric(PARTITION_METRIC_MESSAGES, Math.max(0, entry.getValue() - beginOffset) * 1.0f);
|
||||
metricsMap.put(entry.getKey().partition(), metrics);
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn(
|
||||
"class=PartitionMetricServiceImpl||method=getOffsetRelevantMetrics||clusterPhyId={}||topicName={}||endResultMsg={}||beginResultMsg={}||msg=get messages failed",
|
||||
clusterPhyId, topicName, endOffsetMapResult.getMessage(), beginOffsetMapResult.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
return Result.buildSuc(new ArrayList<>(metricsMap.values()));
|
||||
}
|
||||
|
||||
private Result<List<PartitionMetrics>> getMetricFromJmx(VersionItemParam param) {
|
||||
TopicMetricParam metricParam = (TopicMetricParam) param;
|
||||
String metricName = metricParam.getMetric();
|
||||
String topicName = metricParam.getTopic();
|
||||
Long clusterPhyId = metricParam.getClusterId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = this.getJMXInfo(clusterPhyId, metricName);
|
||||
if(null == jmxInfo) {
|
||||
return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);
|
||||
}
|
||||
|
||||
List<PartitionMetrics> metricsList = new ArrayList<>();
|
||||
|
||||
List<Partition> partitionList = partitionService.listPartitionFromCacheFirst(clusterPhyId, topicName);
|
||||
for (Partition partition: partitionList) {
|
||||
if (KafkaConstant.NO_LEADER.equals(partition.getLeaderBrokerId())) {
|
||||
// 2、没有leader，则直接忽略
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3、获取jmx连接
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterPhyId, partition.getLeaderBrokerId());
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// 4、获取jmx指标
|
||||
String value = jmxConnectorWrap.getAttribute(
|
||||
new ObjectName(jmxInfo.getJmxObjectName() + ",topic=" + topicName + ",partition=" + partition.getPartitionId()),
|
||||
jmxInfo.getJmxAttribute()
|
||||
).toString();
|
||||
|
||||
PartitionMetrics metrics = new PartitionMetrics(clusterPhyId, topicName, partition.getLeaderBrokerId(), partition.getPartitionId());
|
||||
metrics.putMetric(metricName, Float.valueOf(value));
|
||||
metricsList.add(metrics);
|
||||
|
||||
} catch (InstanceNotFoundException e) {
|
||||
// ignore
|
||||
continue;
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(
|
||||
"class=PartitionMetricServiceImpl||method=getMetricFromJmx||clusterPhyId={}||topicName={}||partitionId={}||leaderBrokerId={}||metricName={}||msg={}",
|
||||
clusterPhyId, topicName, partition.getPartitionId(), partition.getLeaderBrokerId(), metricName, e.getClass().getName()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(metricsList);
|
||||
}
|
||||
|
||||
private Result<List<PartitionMetrics>> getTopicAvgMetricFromJmx(VersionItemParam param) {
|
||||
TopicMetricParam metricParam = (TopicMetricParam) param;
|
||||
String metricName = metricParam.getMetric();
|
||||
String topicName = metricParam.getTopic();
|
||||
Long clusterPhyId = metricParam.getClusterId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = this.getJMXInfo(clusterPhyId, metricName);
|
||||
if(null == jmxInfo) {
|
||||
return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);
|
||||
}
|
||||
|
||||
List<PartitionMetrics> metricsList = new ArrayList<>();
|
||||
|
||||
List<Partition> partitionList = partitionService.listPartitionFromCacheFirst(clusterPhyId, topicName);
|
||||
for (Partition partition: partitionList) {
|
||||
if (KafkaConstant.NO_LEADER.equals(partition.getLeaderBrokerId())) {
|
||||
// 2、没有leader，则直接忽略
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3、获取jmx连接
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterPhyId, partition.getLeaderBrokerId());
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// 4、获取jmx指标
|
||||
String value = jmxConnectorWrap.getAttribute(new ObjectName(jmxInfo.getJmxObjectName() + ",topic=" + topicName), jmxInfo.getJmxAttribute()).toString();
|
||||
|
||||
Long leaderCount = partitionList.stream().filter(elem -> elem.getLeaderBrokerId().equals(partition.getLeaderBrokerId())).count();
|
||||
if (leaderCount <= 0) {
|
||||
// leader已经切换走了
|
||||
continue;
|
||||
}
|
||||
|
||||
PartitionMetrics metrics = new PartitionMetrics(clusterPhyId, topicName, partition.getLeaderBrokerId(), partition.getPartitionId());
|
||||
metrics.putMetric(metricName, Float.valueOf(value) / leaderCount);
|
||||
metricsList.add(metrics);
|
||||
|
||||
} catch (InstanceNotFoundException e) {
|
||||
// ignore
|
||||
continue;
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(
|
||||
"class=PartitionMetricServiceImpl||method=getTopicAvgMetricFromJmx||clusterPhyId={}||topicName={}||partitionId={}||leaderBrokerId={}||metricName={}||msg={}",
|
||||
clusterPhyId, topicName, partition.getPartitionId(), partition.getLeaderBrokerId(), metricName, e.getClass().getName()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return Result.buildSuc(metricsList);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.partition.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.partition.PartitionOffsetParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.partition.PartitionPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.PartitionConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterRunStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionMap;
|
||||
import com.xiaojukeji.know.streaming.km.common.zookeeper.znode.brokers.PartitionState;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaConsumerClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.partition.PartitionDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.TopicPartitionStateZNode;
|
||||
import kafka.zk.TopicPartitionsZNode;
|
||||
import kafka.zk.TopicZNode;
|
||||
import kafka.zk.TopicsZNode;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
||||
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.VC_HANDLE_NOT_EXIST;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.SERVICE_OP_PARTITION;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service("partitionService")
|
||||
public class PartitionServiceImpl extends BaseVersionControlService implements PartitionService {
|
||||
private static final ILog log = LogFactory.getLog(PartitionServiceImpl.class);
|
||||
|
||||
private static final String PARTITION_OFFSET_GET = "getPartitionOffset";
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Autowired
|
||||
private PartitionDAO partitionDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaConsumerClient kafkaConsumerClient;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return SERVICE_OP_PARTITION;
|
||||
}
|
||||
|
||||
private final Cache<String, List<Partition>> partitionsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(90, TimeUnit.SECONDS)
|
||||
.maximumSize(1000)
|
||||
.build();
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(PARTITION_OFFSET_GET, V_0_10_0_0, V_0_10_2_0, "getPartitionOffsetFromKafkaConsumerClient", this::getPartitionOffsetFromKafkaConsumerClient);
|
||||
registerVCHandler(PARTITION_OFFSET_GET, V_0_10_2_0, V_MAX, "getPartitionOffsetFromKafkaAdminClient", this::getPartitionOffsetFromKafkaAdminClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<String, List<Partition>>> listPartitionsFromKafka(ClusterPhy clusterPhy) {
|
||||
if (clusterPhy.getRunState().equals(ClusterRunStateEnum.RUN_ZK.getRunState())) {
|
||||
return this.getPartitionsFromZKClient(clusterPhy);
|
||||
}
|
||||
|
||||
return this.getPartitionsFromAdminClient(clusterPhy);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Partition> listPartitionByCluster(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return this.convert2PartitionList(partitionDAO.selectList(lambdaQueryWrapper));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PartitionPO> listPartitionPOByCluster(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return partitionDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Partition> listPartitionByTopic(Long clusterPhyId, String topicName) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(PartitionPO::getTopicName, topicName);
|
||||
|
||||
return this.convert2PartitionList(partitionDAO.selectList(lambdaQueryWrapper));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Partition> listPartitionFromCacheFirst(Long clusterPhyId, String topicName) {
|
||||
String clusterPhyIdAndTopicKey = MsgConstant.getClusterTopicKey(clusterPhyId, topicName);
|
||||
List<Partition> partitionList = partitionsCache.getIfPresent(clusterPhyIdAndTopicKey);
|
||||
|
||||
if (!ValidateUtils.isNull(partitionList)) {
|
||||
return partitionList;
|
||||
}
|
||||
|
||||
partitionList = this.listPartitionByTopic(clusterPhyId, topicName);
|
||||
partitionsCache.put(clusterPhyIdAndTopicKey, partitionList);
|
||||
return partitionList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Partition getPartitionFromCacheFirst(Long clusterPhyId, String topicName, Integer partitionId) {
|
||||
List<Partition> partitionList = this.listPartitionFromCacheFirst(clusterPhyId, topicName);
|
||||
if (ValidateUtils.isEmptyList(partitionList)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (Partition partition: partitionList) {
|
||||
if (partition.getPartitionId().equals(partitionId)) {
|
||||
return partition;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Partition> listPartitionByBroker(Long clusterPhyId, Integer brokerId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
List<Partition> partitionList = this.convert2PartitionList(partitionDAO.selectList(lambdaQueryWrapper));
|
||||
|
||||
return partitionList.stream().filter(elem -> elem.getAssignReplicaList().contains(brokerId)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Partition getPartitionByTopicAndPartitionId(Long clusterPhyId, String topicName, Integer partitionId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(PartitionPO::getTopicName, topicName);
|
||||
lambdaQueryWrapper.eq(PartitionPO::getPartitionId, partitionId);
|
||||
|
||||
return this.convert2Partition(partitionDAO.selectOne(lambdaQueryWrapper));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getPartitionSizeByClusterId(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return partitionDAO.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getLeaderPartitionSizeByClusterId(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.ne(PartitionPO::getLeaderBrokerId, -1);
|
||||
|
||||
return partitionDAO.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getNoLeaderPartitionSizeByClusterId(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(PartitionPO::getLeaderBrokerId, -1);
|
||||
|
||||
return partitionDAO.selectCount(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, OffsetSpec offsetSpec, Long timestamp) {
|
||||
Map<TopicPartition, OffsetSpec> topicPartitionOffsets = new HashMap<>();
|
||||
this.listPartitionByTopic(clusterPhyId, topicName)
|
||||
.stream()
|
||||
.forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec));
|
||||
|
||||
try {
|
||||
return (Result<Map<TopicPartition, Long>>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicPartitionOffsets, timestamp));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafka(Long clusterPhyId, String topicName, Integer partitionId, OffsetSpec offsetSpec, Long timestamp) {
|
||||
if (partitionId == null) {
|
||||
return this.getPartitionOffsetFromKafka(clusterPhyId, topicName, offsetSpec, timestamp);
|
||||
}
|
||||
|
||||
Map<TopicPartition, OffsetSpec> topicPartitionOffsets = new HashMap<>();
|
||||
this.listPartitionByTopic(clusterPhyId, topicName)
|
||||
.stream()
|
||||
.filter(elem -> elem.getPartitionId().equals(partitionId))
|
||||
.forEach(elem -> topicPartitionOffsets.put(new TopicPartition(topicName, elem.getPartitionId()), offsetSpec));
|
||||
|
||||
try {
|
||||
return (Result<Map<TopicPartition, Long>>) doVCHandler(clusterPhyId, PARTITION_OFFSET_GET, new PartitionOffsetParam(clusterPhyId, topicPartitionOffsets, timestamp));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updatePartitions(Long clusterPhyId, String topicName, List<Partition> kafkaPartitionList, List<PartitionPO> dbPartitionList) {
|
||||
Map<Integer, Partition> partitionMap = kafkaPartitionList.stream().collect(Collectors.toMap(Partition::getPartitionId, Function.identity()));
|
||||
|
||||
// 更新已有的分区
|
||||
for (PartitionPO dbPartitionPO: dbPartitionList) {
|
||||
Partition partition = partitionMap.remove(dbPartitionPO.getPartitionId());
|
||||
if (partition == null) {
|
||||
// 分区不存在，则进行删除
|
||||
partitionDAO.deleteById(dbPartitionPO.getId());
|
||||
continue;
|
||||
}
|
||||
|
||||
PartitionPO presentPartitionPO = this.convert2PartitionPO(partition);
|
||||
presentPartitionPO.setId(dbPartitionPO.getId());
|
||||
partitionDAO.updateById(presentPartitionPO);
|
||||
}
|
||||
|
||||
// 插入新的分区
|
||||
for (Partition partition: partitionMap.values()) {
|
||||
try {
|
||||
partitionDAO.insert(this.convert2PartitionPO(partition));
|
||||
} catch (DuplicateKeyException dke) {
|
||||
// 多台部署，因此可能会存在重复，此时如果是重复的错误，则忽略该错误
|
||||
}
|
||||
}
|
||||
|
||||
return kafkaPartitionList.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deletePartitionsIfNotIn(Long clusterPhyId, Set<String> topicNameSet) {
|
||||
LambdaQueryWrapper<PartitionPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(PartitionPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
try {
|
||||
List<PartitionPO> poList = partitionDAO.selectList(lambdaQueryWrapper);
|
||||
for (PartitionPO po: poList) {
|
||||
if (topicNameSet.contains(po.getTopicName())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 不存在 则删除
|
||||
partitionDAO.deleteById(po.getId());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=deletePartitionsIfNotIn||clusterPhyId={}||msg=delete failed", clusterPhyId, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafkaAdminClient(VersionItemParam itemParam) {
|
||||
PartitionOffsetParam offsetParam = (PartitionOffsetParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(offsetParam.getClusterPhyId());
|
||||
|
||||
ListOffsetsResult listOffsetsResult = adminClient.listOffsets(offsetParam.getTopicPartitionOffsets(), new ListOffsetsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
|
||||
Map<TopicPartition, Long> offsetMap = new HashMap<>();
|
||||
listOffsetsResult.all().get().entrySet().stream().forEach(elem -> offsetMap.put(elem.getKey(), elem.getValue().offset()));
|
||||
|
||||
return Result.buildSuc(offsetMap);
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(offsetParam.getClusterPhyId()));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getPartitionOffsetFromKafkaAdminClient||clusterPhyId={}||errMsg=exception!", offsetParam.getClusterPhyId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Map<TopicPartition, Long>> getPartitionOffsetFromKafkaConsumerClient(VersionItemParam itemParam) {
|
||||
KafkaConsumer<String, String> kafkaConsumer = null;
|
||||
|
||||
PartitionOffsetParam offsetParam = (PartitionOffsetParam) itemParam;
|
||||
try {
|
||||
if (ValidateUtils.isEmptyMap(offsetParam.getTopicPartitionOffsets())) {
|
||||
return Result.buildSuc(new HashMap<>());
|
||||
}
|
||||
|
||||
kafkaConsumer = kafkaConsumerClient.getClient(offsetParam.getClusterPhyId());
|
||||
|
||||
OffsetSpec offsetSpec = new ArrayList<>(offsetParam.getTopicPartitionOffsets().values()).get(0);
|
||||
if (offsetSpec instanceof OffsetSpec.LatestSpec) {
|
||||
return Result.buildSuc(
|
||||
kafkaConsumer.endOffsets(
|
||||
offsetParam.getTopicPartitionOffsets().keySet(),
|
||||
Duration.ofMillis(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (offsetSpec instanceof OffsetSpec.EarliestSpec) {
|
||||
return Result.buildSuc(
|
||||
kafkaConsumer.beginningOffsets(
|
||||
offsetParam.getTopicPartitionOffsets().keySet(),
|
||||
Duration.ofMillis(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (offsetSpec instanceof OffsetSpec.TimestampSpec) {
|
||||
// 按照时间进行查找
|
||||
Map<TopicPartition, Long> timestampMap = new HashMap<>();
|
||||
offsetParam.getTopicPartitionOffsets().entrySet().stream().forEach(elem -> timestampMap.put(elem.getKey(), offsetParam.getTimestamp()));
|
||||
|
||||
Map<TopicPartition, OffsetAndTimestamp> offsetMetadataMap = kafkaConsumer.offsetsForTimes(
|
||||
timestampMap,
|
||||
Duration.ofMillis(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
Map<TopicPartition, Long> offsetMap = new HashMap<>();
|
||||
offsetMetadataMap.entrySet().stream().forEach(elem -> offsetMap.put(elem.getKey(), elem.getValue().offset()));
|
||||
return Result.buildSuc(offsetMap);
|
||||
}
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "OffsetSpec type illegal");
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(offsetParam.getClusterPhyId()));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getPartitionOffsetFromKafkaConsumerClient||clusterPhyId={}||errMsg=exception!", offsetParam.getClusterPhyId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
} finally {
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumerClient.returnClient(offsetParam.getClusterPhyId(), kafkaConsumer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Map<String, List<Partition>>> getPartitionsFromAdminClient(ClusterPhy clusterPhy) {
|
||||
Map<String, List<Partition>> partitionMap = new HashMap<>();
|
||||
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhy.getId());
|
||||
|
||||
// 获取Topic列表
|
||||
ListTopicsResult listTopicsResult = adminClient.listTopics(new ListTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
for (String topicName: listTopicsResult.names().get()) {
|
||||
DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(
|
||||
Arrays.asList(topicName),
|
||||
new DescribeTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
TopicDescription description = describeTopicsResult.all().get().get(topicName);
|
||||
|
||||
partitionMap.put(topicName, PartitionConverter.convert2PartitionList(clusterPhy.getId(), description));
|
||||
}
|
||||
|
||||
return Result.buildSuc(partitionMap);
|
||||
} catch (Exception e) {
|
||||
log.error("class=PartitionServiceImpl||method=getPartitionsFromAdminClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Map<String, List<Partition>>> getPartitionsFromZKClient(ClusterPhy clusterPhy) {
|
||||
Map<String, List<Partition>> partitionMap = new HashMap<>();
|
||||
|
||||
try {
|
||||
List<String> topicNameList = kafkaZKDAO.getChildren(clusterPhy.getId(), TopicsZNode.path(), false);
|
||||
for (String topicName: topicNameList) {
|
||||
PartitionMap zkPartitionMap = kafkaZKDAO.getData(clusterPhy.getId(), TopicZNode.path(topicName), PartitionMap.class);
|
||||
|
||||
List<Partition> partitionList = new ArrayList<>();
|
||||
List<String> partitionIdList = kafkaZKDAO.getChildren(clusterPhy.getId(), TopicPartitionsZNode.path(topicName), false);
|
||||
for (String partitionId: partitionIdList) {
|
||||
PartitionState partitionState = kafkaZKDAO.getData(clusterPhy.getId(), TopicPartitionStateZNode.path(new TopicPartition(topicName, Integer.valueOf(partitionId))), PartitionState.class);
|
||||
|
||||
Partition partition = new Partition();
|
||||
partition.setClusterPhyId(clusterPhy.getId());
|
||||
partition.setTopicName(topicName);
|
||||
partition.setPartitionId(Integer.valueOf(partitionId));
|
||||
partition.setLeaderBrokerId(partitionState.getLeader());
|
||||
partition.setInSyncReplicaList(partitionState.getIsr());
|
||||
partition.setAssignReplicaList(zkPartitionMap.getPartitionAssignReplicas(Integer.valueOf(partitionId)));
|
||||
|
||||
partitionList.add(partition);
|
||||
}
|
||||
|
||||
partitionMap.put(topicName, partitionList);
|
||||
}
|
||||
|
||||
return Result.buildSuc(partitionMap);
|
||||
} catch (Exception e) {
|
||||
log.error("class=PartitionServiceImpl||method=getPartitionsFromZKClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private List<Partition> convert2PartitionList(List<PartitionPO> poList) {
|
||||
if (poList == null) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
List<Partition> partitionList = new ArrayList<>();
|
||||
for (PartitionPO po: poList) {
|
||||
if(null != po){partitionList.add(convert2Partition(po));}
|
||||
}
|
||||
return partitionList;
|
||||
}
|
||||
|
||||
private List<PartitionPO> convert2PartitionPOList(List<Partition> partitionList) {
|
||||
if (partitionList == null) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
List<PartitionPO> poList = new ArrayList<>();
|
||||
for (Partition partition: partitionList) {
|
||||
poList.add(this.convert2PartitionPO(partition));
|
||||
}
|
||||
return poList;
|
||||
}
|
||||
|
||||
private PartitionPO convert2PartitionPO(Partition partition) {
|
||||
if (partition == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
PartitionPO po = new PartitionPO();
|
||||
po.setClusterPhyId(partition.getClusterPhyId());
|
||||
po.setTopicName(partition.getTopicName());
|
||||
po.setPartitionId(partition.getPartitionId());
|
||||
po.setLeaderBrokerId(partition.getLeaderBrokerId());
|
||||
po.setInSyncReplicas(CommonUtils.intList2String(partition.getInSyncReplicaList()));
|
||||
po.setAssignReplicas(CommonUtils.intList2String(partition.getAssignReplicaList()));
|
||||
return po;
|
||||
}
|
||||
|
||||
private Partition convert2Partition(PartitionPO po) {
|
||||
if(null == po){return null;}
|
||||
|
||||
Partition partition = new Partition();
|
||||
partition.setClusterPhyId(po.getClusterPhyId());
|
||||
partition.setTopicName(po.getTopicName());
|
||||
partition.setPartitionId(po.getPartitionId());
|
||||
partition.setLeaderBrokerId(po.getLeaderBrokerId());
|
||||
partition.setInSyncReplicaList(CommonUtils.string2IntList(po.getInSyncReplicas()));
|
||||
partition.setAssignReplicaList(CommonUtils.string2IntList(po.getAssignReplicas()));
|
||||
return partition;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.reassign;
|
||||
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReassignJobDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReplaceReassignJob;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.reassign.ReassignSubJobPO;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ReassignJobService {
|
||||
/**
|
||||
* 创建迁移任务
|
||||
*/
|
||||
Result<Void> create(Long jobId, ReplaceReassignJob dto, String operator);
|
||||
|
||||
/**
|
||||
* 删除迁移任务
|
||||
*/
|
||||
Result<Void> delete(Long jobId, String operator);
|
||||
|
||||
/**
|
||||
* 修改迁移任务
|
||||
*/
|
||||
Result<Void> modify(Long jobId, ReplaceReassignJob replaceReassignJob, String operator);
|
||||
|
||||
Result<Void> modifyThrottle(Long jobId, Long throttleUnitB, String operator);
|
||||
|
||||
/**
|
||||
* 执行迁移任务
|
||||
*/
|
||||
Result<Void> execute(Long jobId, String operator);
|
||||
|
||||
/**
|
||||
* 取消迁移任务
|
||||
*/
|
||||
Result<Void> cancel(Long jobId, String operator);
|
||||
|
||||
/**
|
||||
* 检查迁移任务
|
||||
*/
|
||||
Result<Void> verifyAndUpdateStatue(Long jobId);
|
||||
|
||||
/**
|
||||
* 更新子任务中扩展字段的数据
|
||||
*/
|
||||
Result<Void> getAndUpdateSubJobExtendData(Long jobId);
|
||||
|
||||
/**
|
||||
* 获取迁移任务信息
|
||||
*/
|
||||
List<ReassignSubJobPO> getSubJobsByJobId(Long jobId);
|
||||
|
||||
/**
|
||||
* 获取按照Topic维度聚合的详情
|
||||
*/
|
||||
Result<ReassignJobDetail> getJobDetailsGroupByTopic(Long jobId);
|
||||
|
||||
/**
|
||||
* 依据任务状态或者其中一个任务ID
|
||||
*/
|
||||
Long getOneRunningJobId(Long clusterPhyId);
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.reassign;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.reassign.ExecuteReassignParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignPlan;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ReassignService {
|
||||
/**
|
||||
* 生成迁移计划
|
||||
* @param clusterPhyId 物理集群ID
|
||||
* @param topicName Topic名称
|
||||
* @param partitionIdList 分区ID
|
||||
* @param brokerIdList BrokerID
|
||||
* @param enableRackAwareness 是否rack感知
|
||||
* @return
|
||||
*/
|
||||
Result<ReassignPlan> generateReassignmentJson(Long clusterPhyId,
|
||||
String topicName,
|
||||
List<Integer> partitionIdList,
|
||||
List<Integer> brokerIdList,
|
||||
Boolean enableRackAwareness);
|
||||
|
||||
/**
|
||||
* 生成副本扩缩计划
|
||||
* @param clusterPhyId 物理集群ID
|
||||
* @param topicName Topic名称
|
||||
* @param newReplicaNum 新的副本数
|
||||
* @param brokerIdList BrokerID
|
||||
* @return
|
||||
*/
|
||||
Result<ReassignPlan> generateReplicaChangeReassignmentJson(Long clusterPhyId,
|
||||
String topicName,
|
||||
Integer newReplicaNum,
|
||||
List<Integer> brokerIdList);
|
||||
|
||||
/**
|
||||
* 执行迁移任务
|
||||
* @param executeReassignParam 参数
|
||||
* @return
|
||||
*/
|
||||
Result<Void> executePartitionReassignments(ExecuteReassignParam executeReassignParam);
|
||||
|
||||
/**
|
||||
* 检查迁移信息
|
||||
* @param executeReassignParam 参数
|
||||
* @return
|
||||
*/
|
||||
Result<ReassignResult> verifyPartitionReassignments(ExecuteReassignParam executeReassignParam);
|
||||
|
||||
Result<Void> changReassignmentThrottles(ExecuteReassignParam executeReassignParam);
|
||||
|
||||
Result<Void> parseExecuteAssignmentArgs(Long clusterPhyId, String reassignmentJson);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.reassign;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignExecutionStrategy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignTask;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ReassignStrategyService {
|
||||
/**
|
||||
* 根据均衡策略生成迁移任务详情
|
||||
* @param executionStrategy 均衡策略
|
||||
|
||||
* @return
|
||||
*/
|
||||
Result<List<ReassignTask>> generateReassignmentTask(ReassignExecutionStrategy executionStrategy);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.reassign.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.reassign.ExecuteReassignParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReassignJobDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReassignSubJobExtendData;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReplaceReassignJob;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.job.ReplaceReassignSubJob;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.reassign.ReassignJobPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.reassign.ReassignSubJobPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.ReassignConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.job.JobStatusEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignJobService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.replica.ReplicaMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.ReplicaMetricVersionItems;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.reassign.ReassignJobDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.reassign.ReassignSubJobDAO;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.interceptor.TransactionAspectSupport;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Service
|
||||
public class ReassignJobServiceImpl implements ReassignJobService {
|
||||
private static final ILog log = LogFactory.getLog(ReassignJobServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private ReassignJobDAO reassignJobDAO;
|
||||
|
||||
@Autowired
|
||||
private ReassignSubJobDAO reassignSubJobDAO;
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyService clusterPhyService;
|
||||
|
||||
@Autowired
|
||||
private ReassignService reassignService;
|
||||
|
||||
@Autowired
|
||||
private ReplicaMetricService replicationMetricService;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Result<Void> create(Long jobId, ReplaceReassignJob replaceReassignJob, String creator) {
|
||||
// 检查参数是否合法
|
||||
Result<Void> rv = checkParamLegalAndModifyOriginData(jobId, replaceReassignJob, creator);
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
try {
|
||||
// 数据写入DB
|
||||
reassignJobDAO.addAndSetId(ReassignConverter.convert2ReassignJobPO(jobId, replaceReassignJob, creator));
|
||||
|
||||
List<ReassignSubJobPO> subJobPOList = ReassignConverter.convert2ReassignSubJobPOList(jobId, replaceReassignJob.getSubJobList());
|
||||
subJobPOList.forEach(elem -> reassignSubJobDAO.insert(elem));
|
||||
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
creator,
|
||||
OperationEnum.ADD.getDesc(),
|
||||
ModuleEnum.JOB_KAFKA_REPLICA_REASSIGN.getDesc(),
|
||||
MsgConstant.getReassignJobBizStr(
|
||||
jobId,
|
||||
replaceReassignJob.getClusterPhyId()
|
||||
),
|
||||
ConvertUtil.obj2Json(replaceReassignJob)
|
||||
));
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=create||jobId={}||replaceReassignJob={}||errMsg=exception", jobId, replaceReassignJob, e);
|
||||
|
||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Result<Void> delete(Long jobId, String operator) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
try {
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
if (JobStatusEnum.canNotDeleteJob(jobPO.getStatus())) {
|
||||
// 状态错误，禁止执行
|
||||
return this.buildActionForbidden(jobId, jobPO.getStatus());
|
||||
}
|
||||
|
||||
reassignJobDAO.deleteById(jobId);
|
||||
|
||||
LambdaQueryWrapper<ReassignSubJobPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(ReassignSubJobPO::getJobId, jobId);
|
||||
reassignSubJobDAO.delete(lambdaQueryWrapper);
|
||||
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.DELETE.getDesc(),
|
||||
ModuleEnum.JOB_KAFKA_REPLICA_REASSIGN.getDesc(),
|
||||
MsgConstant.getReassignJobBizStr(
|
||||
jobId,
|
||||
jobPO.getClusterPhyId()
|
||||
)
|
||||
));
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=delete||jobId={}||errMsg=exception", jobId, e);
|
||||
|
||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Result<Void> modify(Long jobId, ReplaceReassignJob replaceReassignJob, String operator) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
try {
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
if (!JobStatusEnum.isWaiting(jobPO.getStatus())) {
|
||||
// 状态错误，禁止执行
|
||||
return this.buildActionForbidden(jobId, jobPO.getStatus());
|
||||
}
|
||||
|
||||
// 如果任务处于待执行中，则可以修改任意数据
|
||||
Result<Void> rv = this.modifyAll(jobPO, replaceReassignJob);
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.JOB_KAFKA_REPLICA_REASSIGN.getDesc(),
|
||||
MsgConstant.getReassignJobBizStr(
|
||||
jobId,
|
||||
jobPO.getClusterPhyId()
|
||||
),
|
||||
ConvertUtil.obj2Json(replaceReassignJob)
|
||||
));
|
||||
|
||||
return rv;
|
||||
} catch (Exception e) {
|
||||
log.error("method=modify||jobId={}||replaceReassignJob={}||errMsg=exception", jobId, replaceReassignJob, e);
|
||||
|
||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> modifyThrottle(Long jobId, Long throttleUnitB, String operator) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
try {
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
if (JobStatusEnum.isFinished(jobPO.getStatus())) {
|
||||
// 状态错误，禁止执行
|
||||
return this.buildActionForbidden(jobId, jobPO.getStatus());
|
||||
}
|
||||
|
||||
// 修改限流值
|
||||
jobPO.setThrottleUnitByte(throttleUnitB);
|
||||
reassignJobDAO.updateById(jobPO);
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.JOB_KAFKA_REPLICA_REASSIGN.getDesc(),
|
||||
MsgConstant.getReassignJobBizStr(
|
||||
jobId,
|
||||
jobPO.getClusterPhyId()
|
||||
),
|
||||
String.format("新的限流值:[%d]", throttleUnitB)
|
||||
));
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyThrottle||jobId={}||throttleUnitB={}||errMsg=exception", jobId, throttleUnitB, e);
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public Result<Void> execute(Long jobId, String operator) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
if (!JobStatusEnum.canExecuteJob(jobPO.getStatus())) {
|
||||
// 状态错误，禁止执行
|
||||
return this.buildActionForbidden(jobId, jobPO.getStatus());
|
||||
}
|
||||
|
||||
// 修改DB状态
|
||||
this.setJobInRunning(jobPO);
|
||||
|
||||
// 执行任务
|
||||
Result<Void> rv = reassignService.executePartitionReassignments(new ExecuteReassignParam(jobPO.getClusterPhyId(), jobPO.getReassignmentJson(), jobPO.getThrottleUnitByte()));
|
||||
if (rv.failed()) {
|
||||
|
||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EXE.getDesc(),
|
||||
ModuleEnum.JOB_KAFKA_REPLICA_REASSIGN.getDesc(),
|
||||
MsgConstant.getReassignJobBizStr(
|
||||
jobId,
|
||||
jobPO.getClusterPhyId()
|
||||
)
|
||||
));
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> cancel(Long jobId, String operator) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
try {
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
if (!JobStatusEnum.canCancelJob(jobPO.getStatus())) {
|
||||
// 状态错误，禁止执行
|
||||
return this.buildActionForbidden(jobId, jobPO.getStatus());
|
||||
}
|
||||
|
||||
this.setJobCanceled(jobPO);
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.CANCEL.getDesc(),
|
||||
ModuleEnum.JOB_KAFKA_REPLICA_REASSIGN.getDesc(),
|
||||
MsgConstant.getReassignJobBizStr(
|
||||
jobId,
|
||||
jobPO.getClusterPhyId()
|
||||
)
|
||||
));
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=cancel||jobId={}||errMsg=exception", jobId, e);
|
||||
|
||||
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> verifyAndUpdateStatue(Long jobId) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
Result<Void> rv = reassignService.changReassignmentThrottles(
|
||||
new ExecuteReassignParam(jobPO.getClusterPhyId(), jobPO.getReassignmentJson(), jobPO.getThrottleUnitByte())
|
||||
);
|
||||
if (rv.failed()) {
|
||||
log.error("method=verifyAndUpdateStatue||jobId={}||result={}||msg=change throttle failed", jobId, rv);
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 检查迁移的结果
|
||||
Result<ReassignResult> rrr = reassignService.verifyPartitionReassignments(
|
||||
new ExecuteReassignParam(jobPO.getClusterPhyId(), jobPO.getReassignmentJson(), jobPO.getThrottleUnitByte())
|
||||
);
|
||||
|
||||
if (rrr.failed()) {
|
||||
log.error("method=verifyAndUpdateStatue||jobId={}||result={}||msg=verify reassignment failed", jobId, rrr);
|
||||
return Result.buildFromIgnoreData(rrr);
|
||||
}
|
||||
|
||||
// 更新任务状态
|
||||
return this.checkAndSetSuccessIfFinished(jobPO, rrr.getData());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> getAndUpdateSubJobExtendData(Long jobId) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
List<ReassignSubJobPO> subJobPOList = this.getSubJobsByJobId(jobId);
|
||||
for (ReassignSubJobPO subJobPO: subJobPOList) {
|
||||
Result<ReassignSubJobExtendData> extendDataResult = this.getReassignSubJobExtendData(subJobPO);
|
||||
if (extendDataResult.failed()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ReassignSubJobPO newSubJobPO = new ReassignSubJobPO();
|
||||
newSubJobPO.setId(subJobPO.getId());
|
||||
newSubJobPO.setExtendData(ConvertUtil.obj2Json(extendDataResult.getData()));
|
||||
reassignSubJobDAO.updateById(newSubJobPO);
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ReassignSubJobPO> getSubJobsByJobId(Long jobId) {
|
||||
if (jobId == null) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
LambdaQueryWrapper<ReassignSubJobPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(ReassignSubJobPO::getJobId, jobId);
|
||||
|
||||
return reassignSubJobDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReassignJobDetail> getJobDetailsGroupByTopic(Long jobId) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
// 获取任务
|
||||
ReassignJobPO jobPO = reassignJobDAO.selectById(jobId);
|
||||
if (jobPO == null) {
|
||||
// 任务不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getJobNotExist(jobId));
|
||||
}
|
||||
|
||||
// 获取子任务
|
||||
List<ReassignSubJobPO> subJobPOList = this.getSubJobsByJobId(jobId);
|
||||
|
||||
// 数据组装并放回
|
||||
return Result.buildSuc(ReassignConverter.convert2ReassignJobDetail(jobPO, subJobPOList));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getOneRunningJobId(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<ReassignJobPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(ReassignJobPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(ReassignJobPO::getStatus, JobStatusEnum.RUNNING.getStatus());
|
||||
|
||||
List<ReassignJobPO> poList = reassignJobDAO.selectList(lambdaQueryWrapper);
|
||||
if (!ValidateUtils.isEmptyList(poList)) {
|
||||
// 默认获取第一个
|
||||
return poList.get(0).getId();
|
||||
}
|
||||
|
||||
// 获取子任务中待执行的任务，避免主任务和子任务状态不一致
|
||||
LambdaQueryWrapper<ReassignSubJobPO> subLambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
subLambdaQueryWrapper.eq(ReassignSubJobPO::getClusterPhyId, clusterPhyId);
|
||||
subLambdaQueryWrapper.eq(ReassignSubJobPO::getStatus, JobStatusEnum.RUNNING.getStatus());
|
||||
List<ReassignSubJobPO> subPOList = reassignSubJobDAO.selectList(subLambdaQueryWrapper);
|
||||
if (ValidateUtils.isEmptyList(subPOList)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return subPOList.get(0).getJobId();
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private Result<Void> modifyAll(ReassignJobPO jobPO, ReplaceReassignJob replaceReassignJob) {
|
||||
// DB中的sub-job
|
||||
Map<TopicPartition, ReassignSubJobPO> tpSubJobMap = new HashMap<>();
|
||||
this.getSubJobsByJobId(jobPO.getId()).forEach(elem ->
|
||||
tpSubJobMap.put(new TopicPartition(elem.getTopicName(), elem.getPartitionId()), elem)
|
||||
);
|
||||
|
||||
// 当前变更的sub-job
|
||||
List<ReassignSubJobPO> subJobPOList = ReassignConverter.convert2ReassignSubJobPOList(jobPO.getId(), replaceReassignJob.getSubJobList());
|
||||
subJobPOList.forEach(elem -> {
|
||||
ReassignSubJobPO dbSubPO = tpSubJobMap.remove(new TopicPartition(elem.getTopicName(), elem.getPartitionId()));
|
||||
if (dbSubPO == null) {
|
||||
// DB中不存在
|
||||
reassignSubJobDAO.insert(elem);
|
||||
}
|
||||
|
||||
// 已存在则进行更新
|
||||
elem.setId(dbSubPO.getId());
|
||||
reassignSubJobDAO.updateById(elem);
|
||||
});
|
||||
|
||||
// 移除被删除的tp
|
||||
for (ReassignSubJobPO subJobPO: tpSubJobMap.values()) {
|
||||
reassignSubJobDAO.deleteById(subJobPO.getId());
|
||||
}
|
||||
|
||||
// 修改Job
|
||||
reassignJobDAO.updateById(ReassignConverter.convert2ReassignJobPO(jobPO.getId(), replaceReassignJob, jobPO.getCreator()));
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> buildActionForbidden(Long jobId, Integer jobStatus) {
|
||||
return Result.buildFromRSAndMsg(
|
||||
ResultStatus.OPERATION_FORBIDDEN,
|
||||
String.format("jobId:[%d] 当前 status:[%s], 不允许被执行", jobId, JobStatusEnum.valueOfStatus(jobStatus))
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
private Result<Void> checkAndSetSuccessIfFinished(ReassignJobPO jobPO, ReassignResult reassignmentResult) {
|
||||
long now = System.currentTimeMillis();
|
||||
|
||||
boolean existNotFinished = false;
|
||||
List<ReassignSubJobPO> subJobPOList = this.getSubJobsByJobId(jobPO.getId());
|
||||
for (ReassignSubJobPO subJobPO: subJobPOList) {
|
||||
if (!reassignmentResult.checkPartitionFinished(subJobPO.getTopicName(), subJobPO.getPartitionId())) {
|
||||
existNotFinished = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 更新状态
|
||||
ReassignSubJobPO newSubJobPO = new ReassignSubJobPO();
|
||||
newSubJobPO.setId(subJobPO.getId());
|
||||
newSubJobPO.setStatus(JobStatusEnum.SUCCESS.getStatus());
|
||||
newSubJobPO.setFinishedTime(new Date(now));
|
||||
reassignSubJobDAO.updateById(newSubJobPO);
|
||||
}
|
||||
|
||||
// 更新任务状态
|
||||
if (!existNotFinished && !reassignmentResult.isPartsOngoing()) {
|
||||
// 当前没有分区处于迁移中， 并且没有任务并不处于执行中
|
||||
ReassignJobPO newJobPO = new ReassignJobPO();
|
||||
newJobPO.setId(jobPO.getId());
|
||||
newJobPO.setStatus(JobStatusEnum.SUCCESS.getStatus());
|
||||
newJobPO.setFinishedTime(new Date(now));
|
||||
reassignJobDAO.updateById(newJobPO);
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> setJobInRunning(ReassignJobPO jobPO) {
|
||||
long now = System.currentTimeMillis();
|
||||
|
||||
// 更新子任务状态
|
||||
List<ReassignSubJobPO> subJobPOList = this.getSubJobsByJobId(jobPO.getId());
|
||||
for (ReassignSubJobPO subJobPO: subJobPOList) {
|
||||
ReassignSubJobPO newSubJobPO = new ReassignSubJobPO();
|
||||
newSubJobPO.setId(subJobPO.getId());
|
||||
newSubJobPO.setStatus(JobStatusEnum.RUNNING.getStatus());
|
||||
newSubJobPO.setStartTime(new Date(now));
|
||||
|
||||
reassignSubJobDAO.updateById(newSubJobPO);
|
||||
}
|
||||
|
||||
// 更新父任务状态
|
||||
ReassignJobPO newJobPO = new ReassignJobPO();
|
||||
newJobPO.setId(jobPO.getId());
|
||||
newJobPO.setStatus(JobStatusEnum.RUNNING.getStatus());
|
||||
newJobPO.setStartTime(new Date(now));
|
||||
reassignJobDAO.updateById(newJobPO);
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
|
||||
private Result<Void> setJobCanceled(ReassignJobPO jobPO) {
|
||||
// 更新子任务状态
|
||||
List<ReassignSubJobPO> subJobPOList = this.getSubJobsByJobId(jobPO.getId());
|
||||
for (ReassignSubJobPO subJobPO: subJobPOList) {
|
||||
ReassignSubJobPO newSubJobPO = new ReassignSubJobPO();
|
||||
newSubJobPO.setId(subJobPO.getId());
|
||||
newSubJobPO.setStatus(JobStatusEnum.CANCELED.getStatus());
|
||||
reassignSubJobDAO.updateById(newSubJobPO);
|
||||
}
|
||||
|
||||
// 更新父任务状态
|
||||
ReassignJobPO newJobPO = new ReassignJobPO();
|
||||
newJobPO.setId(jobPO.getId());
|
||||
newJobPO.setStatus(JobStatusEnum.CANCELED.getStatus());
|
||||
reassignJobDAO.updateById(newJobPO);
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
|
||||
private Result<Void> checkParamLegalAndModifyOriginData(Long jobId, ReplaceReassignJob replaceReassignJob, String creator) {
|
||||
if (jobId == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, MsgConstant.getJobIdCanNotNull());
|
||||
}
|
||||
|
||||
if (ValidateUtils.isBlank(creator)) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "creator不允许为空");
|
||||
}
|
||||
|
||||
String reassignmentJson = ReassignConverter.convert2ReassignmentJson(replaceReassignJob);
|
||||
|
||||
// 检查生成的迁移Json是否合法
|
||||
Result<Void> rv = reassignService.parseExecuteAssignmentArgs(replaceReassignJob.getClusterPhyId(), reassignmentJson);
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 检查集群是否存在
|
||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(replaceReassignJob.getClusterPhyId());
|
||||
if (clusterPhy == null) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getClusterPhyNotExist(replaceReassignJob.getClusterPhyId()));
|
||||
}
|
||||
|
||||
// 集群Broker集合
|
||||
Set<Integer> brokerIdSet = brokerService.listAllBrokersFromDB(clusterPhy.getId()).stream().map(elem -> elem.getBrokerId()).collect(Collectors.toSet());
|
||||
|
||||
// 集群Topic集合
|
||||
Map<String, Topic> topicMap = topicService.listTopicsFromDB(clusterPhy.getId()).stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity()));
|
||||
|
||||
for (ReplaceReassignSubJob subJob: replaceReassignJob.getSubJobList()) {
|
||||
if (!replaceReassignJob.getClusterPhyId().equals(subJob.getClusterPhyId())) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, "一个任务中，不能同时存在不同集群的Topic同时进行迁移");
|
||||
}
|
||||
|
||||
for (Integer brokerId: subJob.getReassignBrokerIdList()) {
|
||||
if (!brokerIdSet.contains(brokerId)) {
|
||||
// Broker不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getBrokerNotExist(subJob.getClusterPhyId(), brokerId));
|
||||
}
|
||||
}
|
||||
|
||||
Topic topic = topicMap.get(subJob.getTopicName());
|
||||
if (topic == null) {
|
||||
// Topic不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(subJob.getClusterPhyId(), subJob.getTopicName()));
|
||||
}
|
||||
|
||||
if (!topic.getPartitionMap().containsKey(subJob.getPartitionId())) {
|
||||
// 分区不存在
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getPartitionNotExist(subJob.getClusterPhyId(), subJob.getTopicName(), subJob.getPartitionId()));
|
||||
}
|
||||
|
||||
subJob.setOriginReplicaNum(topic.getPartitionMap().get(subJob.getPartitionId()).size());
|
||||
subJob.setOriginalBrokerIdList(topic.getPartitionMap().get(subJob.getPartitionId()));
|
||||
}
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<ReassignSubJobExtendData> getReassignSubJobExtendData(ReassignSubJobPO subJobPO) {
|
||||
// 获取分区信息
|
||||
Partition partition = partitionService.getPartitionByTopicAndPartitionId(
|
||||
subJobPO.getClusterPhyId(),
|
||||
subJobPO.getTopicName(),
|
||||
subJobPO.getPartitionId()
|
||||
);
|
||||
|
||||
if (partition == null) {
|
||||
// 分区不存在，直接返回错误
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getPartitionNotExist(subJobPO.getClusterPhyId(),
|
||||
subJobPO.getTopicName(),
|
||||
subJobPO.getPartitionId())
|
||||
);
|
||||
}
|
||||
|
||||
// 获取leader副本
|
||||
Float leaderLogSize = this.getReplicaLogSize(subJobPO.getClusterPhyId(),
|
||||
partition.getLeaderBrokerId(),
|
||||
subJobPO.getTopicName(),
|
||||
subJobPO.getPartitionId()
|
||||
);
|
||||
|
||||
// 获取新增的副本
|
||||
Set<Integer> newReplicas = new HashSet<>(CommonUtils.string2IntList(subJobPO.getReassignBrokerIds()));
|
||||
newReplicas.removeAll(CommonUtils.string2IntList(subJobPO.getOriginalBrokerIds()));
|
||||
|
||||
// 遍历新增的副本，计算当前已经完成的迁移Size
|
||||
Long finishedLogSizeUnitB = 0L;
|
||||
for (Integer brokerId: newReplicas) {
|
||||
Float replicaLogSize = this.getReplicaLogSize(subJobPO.getClusterPhyId(),
|
||||
brokerId,
|
||||
subJobPO.getTopicName(),
|
||||
subJobPO.getPartitionId()
|
||||
);
|
||||
if (replicaLogSize == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
finishedLogSizeUnitB += replicaLogSize.longValue();
|
||||
}
|
||||
|
||||
ReassignSubJobExtendData extendData = ConvertUtil.str2ObjByJson(subJobPO.getExtendData(), ReassignSubJobExtendData.class);
|
||||
extendData.setFinishedReassignLogSizeUnitB(finishedLogSizeUnitB);
|
||||
if (leaderLogSize != null) {
|
||||
extendData.setNeedReassignLogSizeUnitB(leaderLogSize.longValue() * newReplicas.size());
|
||||
}
|
||||
|
||||
// 计算剩余时间
|
||||
if (extendData.getNeedReassignLogSizeUnitB().equals(0L) || JobStatusEnum.isFinished(subJobPO.getStatus())) {
|
||||
extendData.setRemainTimeUnitMs(0L);
|
||||
} else if (extendData.getFinishedReassignLogSizeUnitB().equals(0L)) {
|
||||
// 未知
|
||||
extendData.setRemainTimeUnitMs(null);
|
||||
} else {
|
||||
Long usedTime = System.currentTimeMillis() - subJobPO.getStartTime().getTime();
|
||||
// (需迁移LogSize / 已迁移LogSize) = (总时间 / 已进行时间)
|
||||
extendData.setRemainTimeUnitMs(extendData.getNeedReassignLogSizeUnitB() * usedTime / extendData.getFinishedReassignLogSizeUnitB());
|
||||
}
|
||||
|
||||
// 数据修正
|
||||
if (JobStatusEnum.isFinished(subJobPO.getStatus())) {
|
||||
// 如果任务已经完成了：
|
||||
// 1、则将需迁移LogSize直接设置为需要迁移的LogSize
|
||||
// 2. 将剩余时间设置为0
|
||||
extendData.setFinishedReassignLogSizeUnitB(extendData.getNeedReassignLogSizeUnitB());
|
||||
extendData.setRemainTimeUnitMs(0L);
|
||||
}
|
||||
|
||||
return Result.buildSuc(extendData);
|
||||
}
|
||||
|
||||
private Float getReplicaLogSize(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId) {
|
||||
Result<ReplicationMetrics> replicaMetricsResult = replicationMetricService.collectReplicaMetricsFromKafka(
|
||||
clusterPhyId,
|
||||
topicName,
|
||||
partitionId,
|
||||
brokerId,
|
||||
ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE
|
||||
);
|
||||
|
||||
if (!replicaMetricsResult.hasData()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return replicaMetricsResult.getData().getMetric(ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,464 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.reassign.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.reassign.ExecuteReassignParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignPlan;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.ReassignResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.ReassignConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import kafka.admin.ReassignPartitionsCommand;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.utils.Time;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import scala.Tuple2;
|
||||
import scala.collection.Seq;
|
||||
import scala.jdk.javaapi.CollectionConverters;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.VC_HANDLE_NOT_EXIST;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.SERVICE_OP_REASSIGNMENT;
|
||||
|
||||
@Service
|
||||
public class ReassignServiceImpl extends BaseVersionControlService implements ReassignService {
|
||||
private static final ILog log = LogFactory.getLog(ReassignServiceImpl.class);
|
||||
|
||||
private static final String EXECUTE_TASK = "executeTask";
|
||||
private static final String VERIFY_TASK = "verifyTask";
|
||||
private static final String REPLACE_THROTTLE_TASK = "replaceThrottleTask";
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return SERVICE_OP_REASSIGNMENT;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(EXECUTE_TASK, V_0_10_2_0, V_2_6_0, "executeTaskByZKClient", this::executeTaskByZKClient);
|
||||
registerVCHandler(EXECUTE_TASK, V_2_6_0, V_MAX, "executeTaskByKafkaClient", this::executeTaskByKafkaClient);
|
||||
|
||||
registerVCHandler(VERIFY_TASK, V_0_10_2_0, V_2_6_0, "verifyTaskByZKClient", this::verifyTaskByZKClient);
|
||||
registerVCHandler(VERIFY_TASK, V_2_6_0, V_MAX, "verifyTaskByKafkaClient", this::verifyTaskByKafkaClient);
|
||||
|
||||
registerVCHandler(REPLACE_THROTTLE_TASK, V_0_10_2_0, V_2_6_0, "modifyThrottleTaskByZKClient", this::modifyThrottleTaskByZKClient);
|
||||
registerVCHandler(REPLACE_THROTTLE_TASK, V_2_6_0, V_MAX, "modifyThrottleTaskByKafkaClient", this::modifyThrottleTaskByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReassignPlan> generateReassignmentJson(Long clusterPhyId,
|
||||
String topicName,
|
||||
List<Integer> partitionIdList,
|
||||
List<Integer> brokerIdList,
|
||||
Boolean enableRackAwareness) {
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhyId);
|
||||
|
||||
// 生成原始的迁移json-map, 生成迁移计划时，会自动严格检查参数
|
||||
Tuple2<scala.collection.Map<TopicPartition, Seq<Object>>, scala.collection.Map<TopicPartition, Seq<Object>>> scalaReassignmentMapTuple =
|
||||
ReassignPartitionsCommand.generateAssignment(
|
||||
adminClient,
|
||||
this.generateTopicMoveJson(topicName),
|
||||
ConvertUtil.list2String(brokerIdList, Constant.COMMA),
|
||||
enableRackAwareness
|
||||
);
|
||||
|
||||
// 过滤掉不需要的分区并进行格式转换
|
||||
Map<TopicPartition, List<Integer>> reAssignPlanMap = this.filterAndConvertReassignmentMap(CollectionConverters.asJava(scalaReassignmentMapTuple._1), partitionIdList);
|
||||
|
||||
Map<TopicPartition, List<Integer>> currentAssignMap = this.filterAndConvertReassignmentMap(CollectionConverters.asJava(scalaReassignmentMapTuple._2), partitionIdList);
|
||||
|
||||
// 返回结果
|
||||
return Result.buildSuc(new ReassignPlan(clusterPhyId, topicName, reAssignPlanMap, currentAssignMap));
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=generateReassignmentJson||clusterPhyId={}||topicName={}||errMsg=not exist error", clusterPhyId, topicName, nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=generateReassignmentJson||clusterPhyId={}||topicName={}||brokerIdList={}||errMsg=exception", clusterPhyId, topicName, brokerIdList, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReassignPlan> generateReplicaChangeReassignmentJson(Long clusterPhyId,
|
||||
String topicName,
|
||||
Integer newReplicaNum,
|
||||
List<Integer> brokerIdList) {
|
||||
try {
|
||||
// 获取Topic当前的分布
|
||||
Map<Integer, List<Integer>> currentPartitionMap = topicService.getTopicPartitionMapFromKafka(clusterPhyId, topicName);
|
||||
|
||||
Map<TopicPartition, List<Integer>> currentAssignMap = new HashMap<>();
|
||||
currentPartitionMap.entrySet().stream().forEach(entry -> currentAssignMap.put(new TopicPartition(topicName, entry.getKey()), entry.getValue()));
|
||||
|
||||
// 获取扩缩副本之后的分配规则
|
||||
Map<TopicPartition, List<Integer>> reAssignPlanMap = this.generateReplicaChangeReassignmentJson(
|
||||
topicName,
|
||||
newReplicaNum,
|
||||
brokerIdList,
|
||||
currentPartitionMap
|
||||
);
|
||||
|
||||
// 返回结果
|
||||
return Result.buildSuc(new ReassignPlan(clusterPhyId, topicName, reAssignPlanMap, currentAssignMap));
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=generateReplicaChangeReassignmentJson||clusterPhyId={}||topicName={}||errMsg=not exist error", clusterPhyId, topicName, nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=generateReplicaChangeReassignmentJson||clusterPhyId={}||topicName={}||brokerIdList={}||errMsg=exception", clusterPhyId, topicName, brokerIdList, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> executePartitionReassignments(ExecuteReassignParam executeReassignParam) {
|
||||
try {
|
||||
return (Result<Void>) doVCHandler(executeReassignParam.getClusterPhyId(), EXECUTE_TASK, executeReassignParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReassignResult> verifyPartitionReassignments(ExecuteReassignParam executeReassignParam) {
|
||||
try {
|
||||
return (Result<ReassignResult>) doVCHandler(executeReassignParam.getClusterPhyId(), VERIFY_TASK, executeReassignParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> changReassignmentThrottles(ExecuteReassignParam executeReassignParam) {
|
||||
try {
|
||||
return (Result<Void>) doVCHandler(executeReassignParam.getClusterPhyId(), REPLACE_THROTTLE_TASK, executeReassignParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> parseExecuteAssignmentArgs(Long clusterPhyId, String reassignmentJson) {
|
||||
try {
|
||||
ReassignPartitionsCommand.parseExecuteAssignmentArgs(reassignmentJson);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (Exception e) {
|
||||
log.error("method=parseExecuteAssignmentArgs||clusterPhyId={}||reassignmentJson={}||errMsg=exception", clusterPhyId, reassignmentJson, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.PARAM_ILLEGAL, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private Result<Void> modifyThrottleTaskByZKClient(VersionItemParam itemParam) {
|
||||
ExecuteReassignParam param = (ExecuteReassignParam) itemParam;
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(param.getClusterPhyId());
|
||||
|
||||
// 迁移Json的迁移信息
|
||||
scala.collection.Map<TopicPartition, Seq<Object>> proposedParts = ReassignPartitionsCommand.parseExecuteAssignmentArgs(param.getReassignmentJson())._1;
|
||||
|
||||
ReassignPartitionsCommand.verifyReplicasAndBrokersInAssignment(kafkaZkClient, proposedParts);
|
||||
|
||||
// 当前ZK上的迁移信息
|
||||
scala.collection.Map<TopicPartition, Seq<Object>> currentParts = kafkaZkClient.getReplicaAssignmentForTopics(
|
||||
proposedParts.keySet().map(elem -> elem.topic()).toSet()
|
||||
);
|
||||
|
||||
// 转为moveMap格式
|
||||
scala.collection.mutable.Map<String, scala.collection.mutable.Map<Object, ReassignPartitionsCommand.PartitionMove>> moveMap =
|
||||
ReassignPartitionsCommand.calculateProposedMoveMap(new scala.collection.mutable.HashMap<>(), proposedParts, currentParts);
|
||||
|
||||
// 对Topic进行限流
|
||||
scala.collection.Map<String, String> leaderThrottles = ReassignPartitionsCommand.calculateLeaderThrottles(moveMap);
|
||||
scala.collection.Map<String, String> followerThrottles = ReassignPartitionsCommand.calculateFollowerThrottles(moveMap);
|
||||
ReassignPartitionsCommand.modifyTopicThrottles(kafkaZkClient, leaderThrottles, followerThrottles);
|
||||
|
||||
// 对Broker进行限流
|
||||
scala.collection.immutable.Set<Object> reassigningBrokers = ReassignPartitionsCommand.calculateReassigningBrokers(moveMap);
|
||||
ReassignPartitionsCommand.modifyBrokerThrottles(kafkaZkClient, reassigningBrokers, param.getThrottleUnitB());
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=modifyThrottleTaskByZKClient||clusterPhyId={}||errMsg=not exist error", param.getClusterPhyId(), nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyThrottleTaskByZKClient||clusterPhyId={}||param={}||errMsg=exception", param.getClusterPhyId(), param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> modifyThrottleTaskByKafkaClient(VersionItemParam itemParam) {
|
||||
ExecuteReassignParam param = (ExecuteReassignParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
ReassignPartitionsCommand.executeAssignment(
|
||||
adminClient,
|
||||
true,
|
||||
param.getReassignmentJson(),
|
||||
param.getThrottleUnitB(),
|
||||
-1L,
|
||||
KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS,
|
||||
Time.SYSTEM
|
||||
);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=modifyThrottleTaskByKafkaClient||clusterPhyId={}||errMsg=not exist error", param.getClusterPhyId(), nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyThrottleTaskByKafkaClient||clusterPhyId={}||param={}||errMsg=exception", param.getClusterPhyId(), param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> executeTaskByZKClient(VersionItemParam itemParam) {
|
||||
ExecuteReassignParam param = (ExecuteReassignParam) itemParam;
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(param.getClusterPhyId());
|
||||
|
||||
ReassignPartitionsCommand.executeAssignment(kafkaZkClient, param.getReassignmentJson(), param.getThrottleUnitB());
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=executeTaskByZKClient||clusterPhyId={}||errMsg=not exist error", param.getClusterPhyId(), nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=executeTaskByZKClient||clusterPhyId={}||param={}||errMsg=exception", param.getClusterPhyId(), param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<Void> executeTaskByKafkaClient(VersionItemParam itemParam) {
|
||||
ExecuteReassignParam param = (ExecuteReassignParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
ReassignPartitionsCommand.executeAssignment(
|
||||
adminClient,
|
||||
false,
|
||||
param.getReassignmentJson(),
|
||||
param.getThrottleUnitB(),
|
||||
-1L,
|
||||
KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS,
|
||||
Time.SYSTEM
|
||||
);
|
||||
|
||||
return Result.buildSuc();
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=executeTaskByKafkaClient||clusterPhyId={}||errMsg=not exist error", param.getClusterPhyId(), nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=executeTaskByKafkaClient||clusterPhyId={}||param={}||errMsg=exception", param.getClusterPhyId(), param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<ReassignResult> verifyTaskByZKClient(VersionItemParam itemParam) {
|
||||
ExecuteReassignParam param = (ExecuteReassignParam) itemParam;
|
||||
try {
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(param.getClusterPhyId());
|
||||
|
||||
ReassignPartitionsCommand.VerifyAssignmentResult assignmentResult = ReassignPartitionsCommand.verifyAssignment(
|
||||
kafkaZkClient,
|
||||
param.getReassignmentJson(),
|
||||
false
|
||||
);
|
||||
|
||||
return Result.buildSuc(ReassignConverter.convert2ReassignmentResult(assignmentResult));
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=verifyTaskByZKClient||clusterPhyId={}||errMsg=not exist error", param.getClusterPhyId(), nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=verifyTaskByZKClient||clusterPhyId={}||param={}||errMsg=exception", param.getClusterPhyId(), param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<ReassignResult> verifyTaskByKafkaClient(VersionItemParam itemParam) {
|
||||
ExecuteReassignParam param = (ExecuteReassignParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
ReassignPartitionsCommand.VerifyAssignmentResult assignmentResult = ReassignPartitionsCommand.verifyAssignment(
|
||||
adminClient,
|
||||
param.getReassignmentJson(),
|
||||
false
|
||||
);
|
||||
|
||||
return Result.buildSuc(ReassignConverter.convert2ReassignmentResult(assignmentResult));
|
||||
} catch (NotExistException nee) {
|
||||
log.error("method=verifyTaskByKafkaClient||clusterPhyId={}||errMsg=not exist error", param.getClusterPhyId(), nee);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("method=verifyTaskByKafkaClient||clusterPhyId={}||param={}||errMsg=exception", param.getClusterPhyId(), param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<TopicPartition, List<Integer>> filterAndConvertReassignmentMap(Map<TopicPartition, Seq<Object>> rowScalaReassignmentMap, List<Integer> partitionIdList) {
|
||||
Map<TopicPartition, List<Integer>> reassignmentMap = new HashMap<>();
|
||||
|
||||
rowScalaReassignmentMap.entrySet()
|
||||
.stream()
|
||||
.filter(
|
||||
// 过滤掉不需要的分区
|
||||
entry1 -> ValidateUtils.isEmptyList(partitionIdList) || partitionIdList.contains(entry1.getKey().partition()))
|
||||
.forEach(
|
||||
// 转换格式
|
||||
entry2 -> reassignmentMap.put(
|
||||
entry2.getKey(),
|
||||
CollectionConverters.asJava(entry2.getValue()).stream().map(brokerId -> (Integer)brokerId).collect(Collectors.toList())
|
||||
)
|
||||
);
|
||||
|
||||
return reassignmentMap;
|
||||
}
|
||||
|
||||
private String generateTopicMoveJson(String topicName) {
|
||||
Map<String, Object> topicNameMap = new HashMap<>(1);
|
||||
topicNameMap.put("topic", topicName);
|
||||
|
||||
Properties properties = new Properties();
|
||||
properties.put("topics", Arrays.asList(topicNameMap));
|
||||
|
||||
properties.put("version", KafkaConstant.DATA_VERSION_ONE);
|
||||
|
||||
return ConvertUtil.obj2Json(properties);
|
||||
}
|
||||
|
||||
private Map<TopicPartition, List<Integer>> generateReplicaChangeReassignmentJson(String topicName,
|
||||
Integer newReplicaNum,
|
||||
List<Integer> brokerIdList,
|
||||
Map<Integer, List<Integer>> currentPartitionMap) throws AdminOperateException{
|
||||
Map<Integer, Integer> brokerIdReplicasMap = this.initBrokerUsed(currentPartitionMap, brokerIdList);
|
||||
|
||||
Map<TopicPartition, List<Integer>> finalAssignmentMap = new HashMap<>();
|
||||
for (Map.Entry<Integer, List<Integer>> entry: currentPartitionMap.entrySet()) {
|
||||
// 获取变更的副本数
|
||||
Integer changedReplicaNum = newReplicaNum - entry.getValue().size();
|
||||
|
||||
if (changedReplicaNum > 0) {
|
||||
// 扩副本
|
||||
finalAssignmentMap.put(
|
||||
new TopicPartition(topicName, entry.getKey()),
|
||||
this.generatePartitionAddReplicaAssignment(entry.getValue(), brokerIdList, brokerIdReplicasMap, changedReplicaNum)
|
||||
);
|
||||
} else if (changedReplicaNum < 0) {
|
||||
// 缩副本，直接缩小末尾几个副本
|
||||
finalAssignmentMap.put(
|
||||
new TopicPartition(topicName, entry.getKey()),
|
||||
entry.getValue().subList(0, entry.getValue().size() + changedReplicaNum)
|
||||
);
|
||||
} else {
|
||||
// 没有变化
|
||||
finalAssignmentMap.put(new TopicPartition(topicName, entry.getKey()), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
return finalAssignmentMap;
|
||||
}
|
||||
|
||||
private List<Integer> generatePartitionAddReplicaAssignment(List<Integer> presentBrokerIdList,
|
||||
List<Integer> assignBrokerIdList,
|
||||
Map<Integer, Integer> brokerIdReplicasMap,
|
||||
Integer changedReplicaNum) throws AdminOperateException{
|
||||
List<Integer> finalBrokerIdList = new ArrayList<>(presentBrokerIdList);
|
||||
|
||||
for (int idx = 0; idx < changedReplicaNum; ++idx) {
|
||||
Integer targetBrokerId = null;
|
||||
for (Integer brokerId: brokerIdReplicasMap.keySet()) {
|
||||
if (finalBrokerIdList.contains(brokerId) || !assignBrokerIdList.contains(brokerId)) {
|
||||
// 如果当前副本已经落在该broker上， 或者 当前broker不在目标broker里面，则跳过该broker
|
||||
continue;
|
||||
}
|
||||
|
||||
if (targetBrokerId == null) {
|
||||
targetBrokerId = brokerId;
|
||||
}
|
||||
|
||||
if (brokerIdReplicasMap.get(targetBrokerId) > brokerIdReplicasMap.get(brokerId)) {
|
||||
// targetBroker副本数 > 当前broker副本数，则用当前broker替换targetBroker
|
||||
targetBrokerId = brokerId;
|
||||
}
|
||||
}
|
||||
|
||||
if (targetBrokerId == null) {
|
||||
// 没有找到合适的broker，则直接抛出异常
|
||||
throw new AdminOperateException("find targetBrokerId failed, maybe brokers not enough");
|
||||
}
|
||||
|
||||
finalBrokerIdList.add(targetBrokerId);
|
||||
|
||||
// 该broker拥有的副本数+1
|
||||
Integer replicas = brokerIdReplicasMap.get(targetBrokerId);
|
||||
brokerIdReplicasMap.put(targetBrokerId, replicas + 1);
|
||||
}
|
||||
|
||||
return finalBrokerIdList;
|
||||
}
|
||||
|
||||
private Map<Integer, Integer> initBrokerUsed(Map<Integer, List<Integer>> currentPartitionMap, List<Integer> brokerIdList) {
|
||||
Map<Integer, Integer> brokerIdReplicasMap = new HashMap<>();
|
||||
|
||||
currentPartitionMap.entrySet().stream().forEach(entry -> {
|
||||
for (Integer brokerId: entry.getValue()) {
|
||||
Integer replicas = brokerIdReplicasMap.getOrDefault(brokerId, 0);
|
||||
brokerIdReplicasMap.put(brokerId, replicas + 1);
|
||||
}
|
||||
});
|
||||
|
||||
brokerIdList.stream().forEach(brokerId -> brokerIdReplicasMap.putIfAbsent(brokerId, 0));
|
||||
|
||||
return brokerIdReplicasMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.reassign.impl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignExecutionStrategy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReassignTask;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.reassign.strategy.ReplaceReassignSub;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.reassign.ReassignStrategyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.replica.ReplicaMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.ReplicaMetricVersionItems;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class ReassignStrategyServiceImpl implements ReassignStrategyService {
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyService clusterPhyService;
|
||||
|
||||
@Autowired
|
||||
private ReplicaMetricService replicationMetricService;
|
||||
|
||||
@Override
|
||||
public Result<List<ReassignTask>> generateReassignmentTask(ReassignExecutionStrategy executionStrategy) {
|
||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(executionStrategy.getClusterPhyId());
|
||||
if (clusterPhy == null){
|
||||
return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST);
|
||||
}
|
||||
if(executionStrategy.getReplaceReassignSubs().isEmpty()){
|
||||
return Result.buildFrom(ResultStatus.PARAM_ILLEGAL);
|
||||
}
|
||||
|
||||
//根据副本大小进行排序
|
||||
List<ReplaceReassignSub> sortSubs = sortByReplaceLogSize(executionStrategy);
|
||||
|
||||
//根据策略生成新的迁移任务
|
||||
List<ReassignTask> reassignTasks = generateReassignmentTask(
|
||||
executionStrategy.getParallelNum(),
|
||||
sortSubs);
|
||||
return Result.buildSuc(reassignTasks);
|
||||
}
|
||||
|
||||
private List<ReassignTask> generateReassignmentTask(Integer parallelNum, List<ReplaceReassignSub> sortSubs){
|
||||
List<ReassignTask> reassignTasks = new ArrayList<>();
|
||||
|
||||
//统计当前任务broker的并行度
|
||||
Map<Integer,Integer> bParallelNum = new HashMap<>();
|
||||
sortSubs.forEach(sub->{
|
||||
Integer leader = sub.getOriginalBrokerIdList().get(0);
|
||||
// 获取需要移入的副本
|
||||
Set<Integer> newReplicas = new HashSet<>(sub.getReassignBrokerIdList());
|
||||
newReplicas.removeAll(sub.getOriginalBrokerIdList());
|
||||
if (newReplicas == null || newReplicas.size() == 0){
|
||||
//只切换leader
|
||||
reassignTasks.add(new ReassignTask(sub.getTopicName(), sub.getPartitionId(), sub.getReassignBrokerIdList()));
|
||||
return;
|
||||
}
|
||||
|
||||
//若分区的所有副本都未计入并行度中，则遵循一个分区的任务不拆封的原则，不管是否超过并行数都加入到任务
|
||||
ReassignTask task = getFirstTaskByReplace(bParallelNum, parallelNum, newReplicas, sub);
|
||||
if (task != null){
|
||||
//更新当前任务并行度
|
||||
modifyParallelNum(bParallelNum, newReplicas, leader);
|
||||
reassignTasks.add(task);
|
||||
return;
|
||||
}
|
||||
|
||||
//是否满足并行度要求
|
||||
if(!checkParallelNum(bParallelNum, newReplicas, parallelNum, leader)){
|
||||
return;
|
||||
}
|
||||
//更新当前任务并行度
|
||||
modifyParallelNum(bParallelNum, newReplicas, leader);
|
||||
reassignTasks.add(new ReassignTask(sub.getTopicName(), sub.getPartitionId(), sub.getReassignBrokerIdList()));
|
||||
});
|
||||
|
||||
return reassignTasks;
|
||||
}
|
||||
|
||||
private Boolean checkParallelNum(Map<Integer,Integer> bParallelNum,
|
||||
Set<Integer> newReplicas,
|
||||
Integer parallelNum,
|
||||
Integer leader){
|
||||
if(bParallelNum.getOrDefault(leader, 0) + newReplicas.size() > parallelNum){
|
||||
return Boolean.FALSE;
|
||||
}
|
||||
for (Integer brokerId : newReplicas){
|
||||
Integer brokerParallelNum = bParallelNum.getOrDefault(brokerId, 0);
|
||||
if (brokerParallelNum + 1 > parallelNum){
|
||||
return Boolean.FALSE;
|
||||
}
|
||||
}
|
||||
return Boolean.TRUE;
|
||||
}
|
||||
|
||||
private ReassignTask getFirstTaskByReplace(Map<Integer,Integer> bParallelNum,
|
||||
Integer parallelNum,
|
||||
Set<Integer> newReplicas,
|
||||
ReplaceReassignSub sub){
|
||||
Integer leader = sub.getReassignBrokerIdList().get(0);
|
||||
|
||||
if (bParallelNum.get(leader) != null){
|
||||
return null;
|
||||
}
|
||||
//若分区的所有副本都未计入并行度中，则遵循一个分区的任务不拆封的原则，不管是否超过并行数都加入到任务
|
||||
for(Integer brokerId : newReplicas) {
|
||||
Integer brokerParallelNum = bParallelNum.get(brokerId);
|
||||
if (brokerParallelNum != null){
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return new ReassignTask(sub.getTopicName(), sub.getPartitionId(), sub.getReassignBrokerIdList());
|
||||
|
||||
}
|
||||
|
||||
private Result<String> modifyParallelNum(Map<Integer,Integer> bParallelNum, Set<Integer> newReplicas, Integer leader){
|
||||
bParallelNum.put(leader, bParallelNum.getOrDefault(leader, 0) + newReplicas.size());
|
||||
newReplicas.forEach(brokerId -> {
|
||||
bParallelNum.put(brokerId, bParallelNum.getOrDefault(brokerId, 0) + 1);
|
||||
});
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private List<ReplaceReassignSub> sortByReplaceLogSize(ReassignExecutionStrategy executionStrategy){
|
||||
List<ReplaceReassignSub> reassignSubs = executionStrategy.getReplaceReassignSubs();
|
||||
reassignSubs.forEach(sub -> {
|
||||
sub.setReplaceLogSize(
|
||||
getReplicaLogSize(executionStrategy.getClusterPhyId(),
|
||||
sub.getOriginalBrokerIdList().get(0),
|
||||
sub.getTopicName(),
|
||||
sub.getPartitionId())
|
||||
);
|
||||
});
|
||||
|
||||
//优先最大副本
|
||||
if (executionStrategy.equals(Constant.NUM_ONE)){
|
||||
return reassignSubs.stream().sorted(Comparator.comparing(ReplaceReassignSub::getReplaceLogSize).reversed()).collect(Collectors.toList());
|
||||
}
|
||||
//优先最小副本
|
||||
return reassignSubs.stream().sorted(Comparator.comparing(ReplaceReassignSub::getReplaceLogSize)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private Float getReplicaLogSize(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId) {
|
||||
Result<ReplicationMetrics> replicaMetricsResult = replicationMetricService.collectReplicaMetricsFromKafka(
|
||||
clusterPhyId,
|
||||
topicName,
|
||||
partitionId,
|
||||
brokerId,
|
||||
ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE
|
||||
);
|
||||
|
||||
if (!replicaMetricsResult.hasData()) {
|
||||
return 0F;
|
||||
}
|
||||
|
||||
return replicaMetricsResult.getData().getMetric(ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_SIZE);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.replica;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ReplicaMetricService {
|
||||
|
||||
/**
|
||||
* 从kafka中采集指标
|
||||
*/
|
||||
Result<ReplicationMetrics> collectReplicaMetricsFromKafka(Long clusterId, String topic, Integer partitionId, Integer brokerId, String metric);
|
||||
Result<ReplicationMetrics> collectReplicaMetricsFromKafkaWithCache(Long clusterPhyId, String topic, Integer brokerId, Integer partitionId, String metric);
|
||||
|
||||
/**
|
||||
* 从ES中获取指标
|
||||
*/
|
||||
Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, MetricDTO dto);
|
||||
|
||||
Result<ReplicationMetrics> getLatestMetricsFromES(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, List<String> metricNames);
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.replica.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.ReplicationMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.ReplicationMetricParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.partition.Partition;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ReplicationMetricPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BeanUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.cache.CollectMetricsLocalCache;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.replica.ReplicaMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.ReplicationMetricESDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.management.InstanceNotFoundException;
|
||||
import javax.management.ObjectName;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_REPLICATION;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_END_OFFSET;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.ReplicaMetricVersionItems.REPLICATION_METRIC_LOG_START_OFFSET;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service
|
||||
public class ReplicaMetricServiceImpl extends BaseMetricService implements ReplicaMetricService {
|
||||
private static final ILog LOGGER = LogFactory.getLog( ReplicaMetricServiceImpl.class);
|
||||
|
||||
public static final String REPLICATION_METHOD_DO_NOTHING = "doNothing";
|
||||
public static final String REPLICATION_METHOD_GET_METRIC_FROM_JMX = "getMetricFromJmx";
|
||||
public static final String REPLICATION_METHOD_GET_IN_SYNC = "getMetricInSync";
|
||||
public static final String REPLICATION_METHOD_GET_METRIC_MESSAGES = "getMetricMessages";
|
||||
|
||||
@Autowired
|
||||
private KafkaJMXClient kafkaJMXClient;
|
||||
|
||||
@Autowired
|
||||
private PartitionService partitionService;
|
||||
|
||||
@Autowired
|
||||
private ReplicationMetricESDAO replicationMetricESDAO;
|
||||
|
||||
@Override
|
||||
protected List<String> listMetricPOFields(){
|
||||
return BeanUtil.listBeanFields(ReplicationMetricPO.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initRegisterVCHandler(){
|
||||
registerVCHandler( REPLICATION_METHOD_DO_NOTHING, this::doNothing);
|
||||
registerVCHandler( REPLICATION_METHOD_GET_METRIC_FROM_JMX, this::getMetricFromJmx);
|
||||
registerVCHandler( REPLICATION_METHOD_GET_METRIC_MESSAGES, this::getMetricMessages);
|
||||
registerVCHandler( REPLICATION_METHOD_GET_IN_SYNC, this::getMetricInSync);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return METRIC_REPLICATION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReplicationMetrics> collectReplicaMetricsFromKafkaWithCache(Long clusterPhyId, String topic,
|
||||
Integer brokerId, Integer partitionId, String metric){
|
||||
Float keyValue = CollectMetricsLocalCache.getReplicaMetrics(clusterPhyId, brokerId, topic, partitionId, metric);
|
||||
if(null != keyValue){
|
||||
ReplicationMetrics replicationMetrics = new ReplicationMetrics(clusterPhyId, topic, partitionId, brokerId);
|
||||
replicationMetrics.putMetric(metric, keyValue);
|
||||
return Result.buildSuc(replicationMetrics);
|
||||
}
|
||||
|
||||
Result<ReplicationMetrics> ret = collectReplicaMetricsFromKafka(clusterPhyId, topic, partitionId, brokerId, metric);
|
||||
if(null == ret || ret.failed() || null == ret.getData()){return ret;}
|
||||
|
||||
// 更新cache
|
||||
ret.getData().getMetrics().entrySet().stream().forEach(
|
||||
metricNameAndValueEntry -> CollectMetricsLocalCache.putReplicaMetrics(
|
||||
clusterPhyId,
|
||||
brokerId,
|
||||
topic,
|
||||
partitionId,
|
||||
metricNameAndValueEntry.getKey(),
|
||||
metricNameAndValueEntry.getValue()
|
||||
)
|
||||
);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReplicationMetrics> collectReplicaMetricsFromKafka(Long clusterId, String topic, Integer partitionId, Integer brokerId, String metric){
|
||||
try {
|
||||
ReplicationMetricParam metricParam = new ReplicationMetricParam(clusterId, topic, brokerId, partitionId, metric);
|
||||
return (Result<ReplicationMetrics>)doVCHandler(clusterId, metric, metricParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, MetricDTO dto) {
|
||||
Map<String/*metric*/, MetricPointVO> metricPointMap = replicationMetricESDAO.getReplicationMetricsPoint(clusterPhyId, topicName, brokerId, partitionId,
|
||||
dto.getMetricsNames(), dto.getAggType(), dto.getStartTime(), dto.getEndTime());
|
||||
|
||||
List<MetricPointVO> metricPoints = new ArrayList<>(metricPointMap.values());
|
||||
return Result.buildSuc(metricPoints);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<ReplicationMetrics> getLatestMetricsFromES(Long clusterPhyId, Integer brokerId, String topicName, Integer partitionId, List<String> metricNames) {
|
||||
ReplicationMetricPO metricPO = replicationMetricESDAO.getReplicationLatestMetrics(clusterPhyId, brokerId, topicName, partitionId, metricNames);
|
||||
return Result.buildSuc(ConvertUtil.obj2Obj(metricPO, ReplicationMetrics.class));
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
private Result<ReplicationMetrics> doNothing(VersionItemParam param) {
|
||||
ReplicationMetricParam metricParam = (ReplicationMetricParam)param;
|
||||
|
||||
return Result.buildSuc(new ReplicationMetrics(metricParam.getClusterPhyId(), metricParam.getTopic(), metricParam.getBrokerId(), metricParam.getPartitionId()));
|
||||
}
|
||||
|
||||
private Result<ReplicationMetrics> getMetricInSync(VersionItemParam param) {
|
||||
ReplicationMetricParam metricParam = (ReplicationMetricParam)param;
|
||||
String metric = metricParam.getMetric();
|
||||
String topic = metricParam.getTopic();
|
||||
Long clusterId = metricParam.getClusterPhyId();
|
||||
Integer brokerId = metricParam.getBrokerId();
|
||||
Integer partitionId = metricParam.getPartitionId();
|
||||
|
||||
Partition partition = partitionService.getPartitionFromCacheFirst(clusterId, topic, partitionId);
|
||||
if(null == partition){
|
||||
return Result.buildFail();
|
||||
}
|
||||
|
||||
List<Integer> inSyncReplicaList = partition.getInSyncReplicaList();
|
||||
Float metricValue = inSyncReplicaList.contains(brokerId) ? 1f : 0f;
|
||||
|
||||
ReplicationMetrics replicationMetrics = new ReplicationMetrics(clusterId, topic, brokerId, partitionId);
|
||||
replicationMetrics.putMetric(metric, metricValue);
|
||||
|
||||
return Result.buildSuc(replicationMetrics);
|
||||
}
|
||||
|
||||
private Result<ReplicationMetrics> getMetricMessages(VersionItemParam param) {
|
||||
ReplicationMetricParam metricParam = (ReplicationMetricParam)param;
|
||||
String metric = metricParam.getMetric();
|
||||
String topic = metricParam.getTopic();
|
||||
Long clusterId = metricParam.getClusterPhyId();
|
||||
Integer brokerId = metricParam.getBrokerId();
|
||||
Integer partitionId = metricParam.getPartitionId();
|
||||
|
||||
Result<ReplicationMetrics> endRet = this.collectReplicaMetricsFromKafkaWithCache(clusterId, topic, brokerId, partitionId, REPLICATION_METRIC_LOG_END_OFFSET);
|
||||
Result<ReplicationMetrics> startRet = this.collectReplicaMetricsFromKafkaWithCache(clusterId, topic, brokerId, partitionId, REPLICATION_METRIC_LOG_START_OFFSET);
|
||||
|
||||
ReplicationMetrics replicationMetrics = new ReplicationMetrics(clusterId, topic, brokerId, partitionId);
|
||||
if(null != endRet && endRet.successful() && null != startRet && startRet.successful()){
|
||||
Float endOffset = endRet.getData().getMetrics().get(REPLICATION_METRIC_LOG_END_OFFSET);
|
||||
Float startOffset = startRet.getData().getMetrics().get(REPLICATION_METRIC_LOG_START_OFFSET);
|
||||
|
||||
replicationMetrics.putMetric(metric, endOffset - startOffset);
|
||||
}
|
||||
|
||||
return Result.buildSuc(replicationMetrics);
|
||||
}
|
||||
|
||||
private Result<ReplicationMetrics> getMetricFromJmx(VersionItemParam param) {
|
||||
ReplicationMetricParam metricParam = (ReplicationMetricParam)param;
|
||||
String metric = metricParam.getMetric();
|
||||
String topic = metricParam.getTopic();
|
||||
Long clusterId = metricParam.getClusterPhyId();
|
||||
Integer brokerId = metricParam.getBrokerId();
|
||||
Integer partitionId = metricParam.getPartitionId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
//2、获取jmx连接
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, brokerId);
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
|
||||
try {
|
||||
//3、获取jmx指标
|
||||
String value = jmxConnectorWrap.getAttribute(new ObjectName(jmxInfo.getJmxObjectName() + ",topic=" + topic + ",partition=" + partitionId),
|
||||
jmxInfo.getJmxAttribute()).toString();
|
||||
|
||||
ReplicationMetrics metrics = new ReplicationMetrics(clusterId, topic, brokerId, partitionId);
|
||||
metrics.putMetric(metric, Float.valueOf(value));
|
||||
|
||||
return Result.buildSuc(metrics);
|
||||
} catch (InstanceNotFoundException e) {
|
||||
return Result.buildFailure(VC_JMX_INSTANCE_NOT_FOUND);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("getMetricFromJmx||cluster={}||brokerId={}||metrics={}||jmx={}||msg={}",
|
||||
clusterId, brokerId, metric, jmxInfo.getJmxObjectName(), e.getClass().getName());
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicCreateParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicPartitionExpandParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
|
||||
public interface OpTopicService {
|
||||
/**
|
||||
* 创建Topic
|
||||
*/
|
||||
Result<Void> createTopic(TopicCreateParam createParam, String operator);
|
||||
|
||||
/**
|
||||
* 删除Topic
|
||||
*/
|
||||
Result<Void> deleteTopic(TopicParam param, String operator);
|
||||
|
||||
/**
|
||||
* 扩分区
|
||||
*/
|
||||
Result<Void> expandTopic(TopicPartitionExpandParam expandParam, String operator);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaConfigDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaTopicConfigParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Kafka相关配置接口
|
||||
* @author zengqiao
|
||||
* @date 22/03/03
|
||||
*/
|
||||
public interface TopicConfigService {
|
||||
/**
|
||||
* 修改Topic配置
|
||||
* @param kafkaTopicConfigParam 修改信息
|
||||
* @param operator 操作人
|
||||
* @return
|
||||
*/
|
||||
Result<Void> modifyTopicConfig(KafkaTopicConfigParam kafkaTopicConfigParam, String operator);
|
||||
|
||||
/**
|
||||
* 从Kafka直接获取Topic配置
|
||||
* @param clusterPhyId 物理集群ID
|
||||
* @param topicName Topic名称
|
||||
* @return
|
||||
*/
|
||||
Result<List<KafkaConfigDetail>> getTopicConfigDetailFromKafka(Long clusterPhyId, String topicName);
|
||||
|
||||
Result<Map<String, String>> getTopicConfigFromKafka(Long clusterPhyId, String topicName);
|
||||
|
||||
List<Properties> getConfigNamesAndDocs(Long clusterPhyId);
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsTopicDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface TopicMetricService {
|
||||
/**
|
||||
* 从 kafka 中获取相应的指标
|
||||
*/
|
||||
Result<List<TopicMetrics>> collectTopicMetricsFromKafka(Long clusterPhyId, String topicName, String metricName);
|
||||
Result<List<TopicMetrics>> collectTopicMetricsFromKafkaWithCacheFirst(Long clusterPhyId, String topicName, String metricName);
|
||||
|
||||
/**
|
||||
* 优先从本地缓存获取metrics信息
|
||||
*/
|
||||
Map<String, TopicMetrics> getLatestMetricsFromCacheFirst(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 获取Topic在具体Broker上最新的一个指标
|
||||
* @param clusterPhyId 物理集群ID
|
||||
* @param brokerId brokerId
|
||||
* @param topicName Topic名称
|
||||
* @param metricNameList 需获取指标列表
|
||||
* @return
|
||||
*/
|
||||
TopicMetrics getTopicLatestMetricsFromES(Long clusterPhyId, Integer brokerId, String topicName, List<String> metricNameList);
|
||||
|
||||
/**
|
||||
* 获取Topic维度最新的一条指标
|
||||
* @param clusterPhyId
|
||||
* @param topicNames
|
||||
* @param metricNameList
|
||||
* @return
|
||||
*/
|
||||
List<TopicMetrics> listTopicLatestMetricsFromES(Long clusterPhyId, List<String> topicNames, List<String> metricNameList);
|
||||
/**
|
||||
* 获取Topic维度最新的一条指标
|
||||
* @param clusterPhyId
|
||||
* @param topicName
|
||||
* @param metricNameList
|
||||
* @return
|
||||
*/
|
||||
TopicMetrics getTopicLatestMetricsFromES(Long clusterPhyId, String topicName, List<String> metricNameList);
|
||||
|
||||
/**
|
||||
* 从 es 中获取聚合计算之后的指标用于页面展示
|
||||
* @param dto
|
||||
* @return
|
||||
*/
|
||||
Result<List<MetricMultiLinesVO>> listTopicMetricsFromES(Long clusterId, MetricsTopicDTO dto);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param clusterId
|
||||
* @param topicName
|
||||
* @param dto
|
||||
* @return
|
||||
*/
|
||||
Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterId, String topicName, MetricDTO dto);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param clusterPhyId
|
||||
* @param topics
|
||||
* @param metric
|
||||
* @param max
|
||||
* @param startTime
|
||||
* @param endTime
|
||||
* @return
|
||||
*/
|
||||
Result<List<TopicMetrics>> listTopicMaxMinMetrics(Long clusterPhyId, List<String> topics, String metric, boolean max, Long startTime, Long endTime);
|
||||
|
||||
/**
|
||||
* 获取聚合指标点
|
||||
* @param clusterPhyId
|
||||
* @param topicNames
|
||||
* @param metricName
|
||||
* @param aggTypeEnum
|
||||
* @param startTime
|
||||
* @param endTime
|
||||
* @return
|
||||
*/
|
||||
Result<Map<String, MetricPointVO>> getAggMetricPointFromES(Long clusterPhyId,
|
||||
List<String> topicNames,
|
||||
String metricName,
|
||||
AggTypeEnum aggTypeEnum,
|
||||
Long startTime,
|
||||
Long endTime);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param clusterId
|
||||
* @param topicName
|
||||
* @param dto
|
||||
* @return
|
||||
*/
|
||||
Result<Map<String, List<MetricPointVO>>> getMetricPointsFromES(Long clusterId, List<String> topicName, MetricDTO dto);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param clusterId
|
||||
* @param metricNameList
|
||||
* @param sort
|
||||
* @param fuzzy
|
||||
* @param searchTermList
|
||||
* @param page
|
||||
* @return
|
||||
*/
|
||||
PaginationResult<TopicMetrics> pagingTopicWithLatestMetricsFromES(Long clusterId, List<String> metricNameList,
|
||||
SearchSort sort,
|
||||
SearchFuzzy fuzzy,
|
||||
List<SearchShould> shoulds,
|
||||
List<SearchTerm> searchTermList,
|
||||
SearchPage page);
|
||||
|
||||
/**
|
||||
* 统计一段时间内，某个指标出现某个值的次数
|
||||
* @param clusterPhyId 集群ID
|
||||
* @param topicName 消费组名称
|
||||
* @param searchMatch 指标
|
||||
* @param startTime 起始时间
|
||||
* @param endTime 结束时间
|
||||
* @return
|
||||
*/
|
||||
Result<Integer> countMetricValueOccurrencesFromES(Long clusterPhyId, String topicName, SearchTerm searchMatch, Long startTime, Long endTime);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.topic.TopicPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface TopicService {
|
||||
/**
|
||||
* 从Kafka获取数据
|
||||
*/
|
||||
Result<List<Topic>> listTopicsFromKafka(ClusterPhy clusterPhy);
|
||||
Map<Integer, List<Integer>> getTopicPartitionMapFromKafka(Long clusterPhyId, String topicName) throws NotExistException, AdminOperateException;
|
||||
|
||||
/**
|
||||
* 从DB获取数据
|
||||
*/
|
||||
List<Topic> listTopicsFromDB(Long clusterPhyId);
|
||||
Topic getTopic(Long clusterPhyId, String topicName);
|
||||
List<String> listRecentUpdateTopicNamesFromDB(Long clusterPhyId, Integer time); // 获取集群最近新增Topic的topic名称：time单位为秒
|
||||
|
||||
/**
|
||||
* 优先从缓存获取数据
|
||||
*/
|
||||
List<Topic> listTopicsFromCacheFirst(Long clusterPhyId);
|
||||
Topic getTopicFromCacheFirst(Long clusterPhyId, String topicName);
|
||||
Integer getTopicSizeFromCacheFirst(Long clusterPhyId);
|
||||
Integer getReplicaSizeFromCacheFirst(Long clusterPhyId);
|
||||
|
||||
/**
|
||||
* 操作DB
|
||||
*/
|
||||
int addNewTopic2DB(TopicPO po);
|
||||
int deleteTopicInDB(Long clusterPhyId, String topicName);
|
||||
void batchReplaceMetadata(Long clusterPhyId, List<Topic> presentTopicList);
|
||||
int batchReplaceConfig(Long clusterPhyId, List<TopicConfig> topicConfigList);
|
||||
Result<Void> updatePartitionNum(Long clusterPhyId, String topicName, Integer partitionNum);
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicCreateParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicPartitionExpandParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.TopicConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.OpTopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.controller.ReplicaAssignment;
|
||||
import kafka.server.ConfigType;
|
||||
import kafka.zk.AdminZkClient;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import scala.Option;
|
||||
import scala.collection.Seq;
|
||||
import scala.jdk.javaapi.CollectionConverters;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.VC_HANDLE_NOT_EXIST;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.SERVICE_OP_TOPIC;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service
|
||||
public class OpTopicServiceImpl extends BaseVersionControlService implements OpTopicService {
|
||||
private static final ILog log = LogFactory.getLog(TopicConfigServiceImpl.class);
|
||||
|
||||
private static final String TOPIC_CREATE = "createTopic";
|
||||
private static final String TOPIC_DELETE = "deleteTopic";
|
||||
private static final String TOPIC_EXPAND = "expandTopic";
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return SERVICE_OP_TOPIC;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(TOPIC_CREATE, V_0_10_0_0, V_0_10_0_1, "createByZKClientV1", this::createByZKClientV1);
|
||||
registerVCHandler(TOPIC_CREATE, V_0_10_0_1, V_0_11_0_3, "createByZKClientV2", this::createByZKClientV2);
|
||||
registerVCHandler(TOPIC_CREATE, V_0_11_0_3, V_MAX, "createByKafkaClient", this::createByKafkaClient);
|
||||
|
||||
registerVCHandler(TOPIC_DELETE, V_0_10_0_0, V_0_11_0_3, "deleteByZKClient", this::deleteByZKClient);
|
||||
registerVCHandler(TOPIC_DELETE, V_0_11_0_3, V_MAX, "deleteByKafkaClient", this::deleteByKafkaClient);
|
||||
|
||||
registerVCHandler(TOPIC_EXPAND, V_0_10_0_0, V_0_11_0_3, "expandTopicByZKClient", this::expandTopicByZKClient);
|
||||
registerVCHandler(TOPIC_EXPAND, V_0_11_0_3, V_MAX, "expandTopicByKafkaClient", this::expandTopicByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> createTopic(TopicCreateParam createParam, String operator) {
|
||||
// 检查参数
|
||||
Result<Void> rv = createParam.simpleCheckFieldIsNull();
|
||||
if (rv.failed()) {
|
||||
return Result.buildFromIgnoreData(rv);
|
||||
}
|
||||
|
||||
// 进行Topic创建
|
||||
try {
|
||||
rv = (Result<Void>) doVCHandler(createParam.getClusterPhyId(), TOPIC_CREATE, createParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
OplogDTO oplogDTO = new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.ADD.getDesc(),
|
||||
ModuleEnum.KAFKA_TOPIC.getDesc(),
|
||||
MsgConstant.getTopicBizStr(createParam.getClusterPhyId(),createParam.getTopicName()),
|
||||
"新增" + createParam.toString());
|
||||
opLogWrapService.saveOplogAndIgnoreException(oplogDTO);
|
||||
|
||||
try {
|
||||
topicService.addNewTopic2DB(TopicConverter.convert2TopicPOAndIgnoreTime(createParam));
|
||||
} catch (Exception e) {
|
||||
log.error("method=createTopic||param={}||operator={}||msg=add topic to db failed||errMsg=exception", createParam, operator, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.MYSQL_OPERATE_FAILED, "Topic创建成功，但记录到DB中失败，将缺少部分业务数据");
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> deleteTopic(TopicParam param, String operator) {
|
||||
try {
|
||||
Result<Void> rv = (Result<Void>) doVCHandler(param.getClusterPhyId(), TOPIC_DELETE, param);
|
||||
if (rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 删除DB中的Topic数据
|
||||
topicService.deleteTopicInDB(param.getClusterPhyId(), param.getTopicName());
|
||||
|
||||
// 记录操作
|
||||
OplogDTO oplogDTO = new OplogDTO(operator,
|
||||
OperationEnum.DELETE.getDesc(),
|
||||
ModuleEnum.KAFKA_TOPIC.getDesc(),
|
||||
MsgConstant.getTopicBizStr(param.getClusterPhyId(), param.getTopicName()),
|
||||
String.format("删除Topic:[%s]", param.toString()));
|
||||
opLogWrapService.saveOplogAndIgnoreException(oplogDTO);
|
||||
return rv;
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> expandTopic(TopicPartitionExpandParam expandParam, String operator) {
|
||||
// 检查参数
|
||||
Result<Void> rv = expandParam.simpleCheckFieldIsNull();
|
||||
if (rv.failed()) {
|
||||
return Result.buildFromIgnoreData(rv);
|
||||
}
|
||||
|
||||
try {
|
||||
rv = (Result<Void>) doVCHandler(expandParam.getClusterPhyId(), TOPIC_EXPAND, expandParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
// 记录操作
|
||||
OplogDTO oplogDTO = new OplogDTO(operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.KAFKA_TOPIC.getDesc(),
|
||||
MsgConstant.getTopicBizStr(expandParam.getClusterPhyId(),expandParam.getTopicName()),
|
||||
MsgConstant.getTopicExtend((long)expandParam.getExistPartitionNum(), (long) expandParam.getTotalPartitionNum(),expandParam.toString()));
|
||||
opLogWrapService.saveOplogAndIgnoreException(oplogDTO);
|
||||
return rv;
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private Result<Void> deleteByKafkaClient(VersionItemParam itemParam) {
|
||||
TopicParam param = (TopicParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(Arrays.asList(param.getTopicName()), new DeleteTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
deleteTopicsResult.all().get();
|
||||
} catch (Exception e) {
|
||||
log.error("delete topic by kafka-client failed，clusterPhyId:{} topicName:{}", param.getClusterPhyId(), param.getTopicName(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> deleteByZKClient(VersionItemParam itemParam){
|
||||
TopicParam param = (TopicParam) itemParam;
|
||||
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(param.getClusterPhyId());
|
||||
|
||||
adminZkClient.deleteTopic(param.getTopicName());
|
||||
} catch (Exception e) {
|
||||
log.error("delete topic by zk-client failed，clusterPhyId:{} topicName:{}", param.getClusterPhyId(), param.getTopicName(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> createByKafkaClient(VersionItemParam itemParam) {
|
||||
TopicCreateParam createParam = (TopicCreateParam)itemParam;
|
||||
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(createParam.getClusterPhyId());
|
||||
|
||||
CreateTopicsResult createTopicsResult = adminClient.createTopics(
|
||||
Arrays.asList(new NewTopic(createParam.getTopicName(), createParam.getAssignmentMap()).configs(createParam.getConfig())),
|
||||
new CreateTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
createTopicsResult.all().get();
|
||||
} catch (Exception e) {
|
||||
log.error("method=createByKafkaClient||param={}||errMsg=exception!", createParam, e);
|
||||
|
||||
return Result.buildFailure(ResultStatus.KAFKA_OPERATE_FAILED);
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> createByZKClientV2(VersionItemParam itemParam){
|
||||
TopicCreateParam createParam = (TopicCreateParam)itemParam;
|
||||
|
||||
// 转换配置的类型
|
||||
Map<String, String> config = createParam.getConfig();
|
||||
Properties props = new Properties();
|
||||
props.putAll(config);
|
||||
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(createParam.getClusterPhyId());
|
||||
|
||||
adminZkClient.createTopicWithAssignment(createParam.getTopicName(), props, this.convert2ScalaMap(createParam.getAssignmentMap()), true, false);
|
||||
} catch (Exception e) {
|
||||
log.error("method=createByZKClientV2||param={}||errMsg=exception!", createParam, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> createByZKClientV1(VersionItemParam itemParam){
|
||||
TopicCreateParam createParam = (TopicCreateParam)itemParam;
|
||||
|
||||
// 配置
|
||||
Map<String, String> config = createParam.getConfig();
|
||||
Properties props = new Properties();
|
||||
props.putAll(config);
|
||||
|
||||
// assignment
|
||||
scala.collection.Map<Object, Seq<Object>> assignmentMap = this.convert2ScalaMap(createParam.getAssignmentMap());
|
||||
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(createParam.getClusterPhyId());
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(createParam.getClusterPhyId());
|
||||
|
||||
// 检查参数是否合法
|
||||
adminZkClient.validateTopicCreate(createParam.getTopicName(), assignmentMap, props);
|
||||
|
||||
// 修改配置的数据节点
|
||||
kafkaZkClient.setOrCreateEntityConfigs(ConfigType.Topic(), createParam.getTopicName(), props);
|
||||
|
||||
// 修改配置的通知节点
|
||||
kafkaZKDAO.createConfigChangeNotificationVersionOne(createParam.getClusterPhyId(), ConfigType.Topic(), createParam.getTopicName());
|
||||
|
||||
// 创建TopicAssignment
|
||||
Map<TopicPartition, Seq<Object>> replicaAssignmentMap = new HashMap<>();
|
||||
for (Map.Entry<Object, Seq<Object>> entry: CollectionConverters.asJava(assignmentMap).entrySet()) {
|
||||
replicaAssignmentMap.put(new TopicPartition(createParam.getTopicName(), (Integer) entry.getKey()), entry.getValue());
|
||||
}
|
||||
|
||||
kafkaZkClient.createTopicAssignment(createParam.getTopicName(), Option.apply(null), CollectionConverters.asScala(replicaAssignmentMap));
|
||||
} catch (Exception e) {
|
||||
log.error("method=createByZKClientV1||param={}||errMsg=exception!", createParam, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> expandTopicByKafkaClient(VersionItemParam itemParam) {
|
||||
TopicPartitionExpandParam expandParam = (TopicPartitionExpandParam) itemParam;
|
||||
try {
|
||||
Map<String, NewPartitions> newPartitionMap = new HashMap<>();
|
||||
newPartitionMap.put(expandParam.getTopicName(), NewPartitions.increaseTo(
|
||||
expandParam.getTotalPartitionNum(),
|
||||
new ArrayList<>(expandParam.getNewPartitionAssignment().values())
|
||||
));
|
||||
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(expandParam.getClusterPhyId());
|
||||
CreatePartitionsResult createPartitionsResult = adminClient.createPartitions(
|
||||
newPartitionMap,
|
||||
new CreatePartitionsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
createPartitionsResult.all().get();
|
||||
} catch (NotExistException nee) {
|
||||
log.warn("method=expandTopicByKafkaClient||param={}||msg=cluster or topic not exist", expandParam);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.warn("method=expandTopicByKafkaClient||param={}||errMsg=exception!", expandParam, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> expandTopicByZKClient(VersionItemParam itemParam){
|
||||
TopicPartitionExpandParam expandParam = (TopicPartitionExpandParam) itemParam;
|
||||
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(expandParam.getClusterPhyId());
|
||||
|
||||
Map<Object, ReplicaAssignment> existingAssignment = new HashMap<>();
|
||||
for (Map.Entry<Integer, List<Integer>> entry: expandParam.getExistingAssignment().entrySet()) {
|
||||
existingAssignment.put(entry.getKey(), ReplicaAssignment.apply(CollectionConverters.asScala(new ArrayList<>(entry.getValue()))));
|
||||
}
|
||||
|
||||
Map<Object, ReplicaAssignment> newPartitionAssignment = new HashMap<>();
|
||||
for (Map.Entry<Integer, List<Integer>> entry: expandParam.getNewPartitionAssignment().entrySet()) {
|
||||
newPartitionAssignment.put(entry.getKey(), ReplicaAssignment.apply(CollectionConverters.asScala(new ArrayList<>(entry.getValue()))));
|
||||
}
|
||||
|
||||
adminZkClient.createPartitionsWithAssignment(
|
||||
expandParam.getTopicName(),
|
||||
CollectionConverters.asScala(existingAssignment),
|
||||
CollectionConverters.asScala(newPartitionAssignment)
|
||||
);
|
||||
} catch (NotExistException nee) {
|
||||
log.warn("method=expandTopicByZKClient||param={}||msg=cluster or topic not exist", expandParam);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, nee.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.warn("method=expandTopicByZKClient||param={}||errMsg={}", expandParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private scala.collection.Map<Object, Seq<Object>> convert2ScalaMap(Map<Integer, List<Integer>> rawAssignmentMap) {
|
||||
Map<Object, Seq<Object>> assignmentMap = new HashMap<>();
|
||||
rawAssignmentMap.entrySet().stream().forEach(elem ->
|
||||
assignmentMap.put(
|
||||
elem.getKey(),
|
||||
CollectionConverters.asScala(elem.getValue().stream().map(item -> (Object)item).collect(Collectors.toList()))
|
||||
)
|
||||
);
|
||||
|
||||
return CollectionConverters.asScala(assignmentMap);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.didiglobal.logi.security.common.dto.oplog.OplogDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.config.kafkaconfig.KafkaConfigDetail;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.config.KafkaTopicConfigParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.topic.TopicParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.kafka.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.KafkaConfigConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.ModuleEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.operaterecord.OperationEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.Triple;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.VersionUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.oprecord.OpLogWrapService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicConfigService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseVersionControlService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminZKClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.server.ConfigType;
|
||||
import kafka.zk.AdminZkClient;
|
||||
import kafka.zk.KafkaZkClient;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.config.ConfigResource;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.*;
|
||||
|
||||
|
||||
@Service
|
||||
public class TopicConfigServiceImpl extends BaseVersionControlService implements TopicConfigService {
|
||||
private static final ILog log = LogFactory.getLog(TopicConfigServiceImpl.class);
|
||||
|
||||
private static final String GET_TOPIC_CONFIG = "getTopicConfig";
|
||||
private static final String MODIFY_TOPIC_CONFIG = "modifyTopicConfig";
|
||||
|
||||
/**
|
||||
* Topic的配置支持的版本信息
|
||||
*/
|
||||
private static final List<Triple<Long/*minSupportVersion*/, Long/*maxSupportVersion*/, List<Properties>>> TOPIC_CONFIG_NAMES_AND_DOCS = Arrays.asList(
|
||||
new Triple<>(0L, V_0_10_1_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig0100.class)),
|
||||
new Triple<>(V_0_10_1_0.getVersionL(), V_0_10_2_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig0101.class)),
|
||||
new Triple<>(V_0_10_2_0.getVersionL(), V_0_11_0_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig0102.class)),
|
||||
|
||||
new Triple<>(V_0_11_0_0.getVersionL(), V_1_0_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig0110.class)),
|
||||
|
||||
new Triple<>(V_1_0_0.getVersionL(), V_1_1_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig1000.class)),
|
||||
new Triple<>(V_1_1_0.getVersionL(), V_2_0_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig1100.class)),
|
||||
|
||||
new Triple<>(V_2_0_0.getVersionL(), V_2_5_0.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig2000.class)),
|
||||
new Triple<>(V_2_5_0.getVersionL(), V_MAX.getVersionL(), AbstractTopicConfig.getTopicConfigNamesAndDocs(TopicConfig2500.class))
|
||||
);
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminZKClient kafkaAdminZKClient;
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyService clusterPhyService;
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Autowired
|
||||
private OpLogWrapService opLogWrapService;
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.SERVICE_OP_TOPIC_CONFIG;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
private void init() {
|
||||
registerVCHandler(GET_TOPIC_CONFIG, V_0_10_0_0, V_0_11_0_0, "getTopicConfigByZKClient", this::getTopicConfigByZKClient);
|
||||
registerVCHandler(GET_TOPIC_CONFIG, V_0_11_0_0, V_MAX, "getTopicConfigByKafkaClient", this::getTopicConfigByKafkaClient);
|
||||
|
||||
registerVCHandler(MODIFY_TOPIC_CONFIG, V_0_10_0_0, V_0_10_2_0, "modifyTopicConfigByZKClientAndNodeVersionV1", this::modifyTopicConfigByZKClientAndNodeVersionV1);
|
||||
registerVCHandler(MODIFY_TOPIC_CONFIG, V_0_10_2_0, V_0_11_0_3, "modifyTopicConfigByZKClientAndNodeVersionV2", this::modifyTopicConfigByZKClientAndNodeVersionV2);
|
||||
registerVCHandler(MODIFY_TOPIC_CONFIG, V_0_11_0_3, V_MAX, "modifyTopicConfigByKafkaClient", this::modifyTopicConfigByKafkaClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> modifyTopicConfig(KafkaTopicConfigParam kafkaTopicConfigParam, String operator) {
|
||||
Result<Void> rv = null;
|
||||
try {
|
||||
rv = (Result<Void>) doVCHandler(kafkaTopicConfigParam.getClusterPhyId(), MODIFY_TOPIC_CONFIG, kafkaTopicConfigParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
|
||||
if (rv == null || rv.failed()) {
|
||||
return rv;
|
||||
}
|
||||
|
||||
// 记录操作
|
||||
opLogWrapService.saveOplogAndIgnoreException(new OplogDTO(
|
||||
operator,
|
||||
OperationEnum.EDIT.getDesc(),
|
||||
ModuleEnum.KAFKA_TOPIC_CONFIG.getDesc(),
|
||||
MsgConstant.getTopicBizStr(kafkaTopicConfigParam.getClusterPhyId(), kafkaTopicConfigParam.getTopicName()),
|
||||
ConvertUtil.obj2Json(kafkaTopicConfigParam)
|
||||
));
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<KafkaConfigDetail>> getTopicConfigDetailFromKafka(Long clusterPhyId, String topicName) {
|
||||
try {
|
||||
return (Result<List<KafkaConfigDetail>>) doVCHandler(clusterPhyId, GET_TOPIC_CONFIG, new TopicParam(clusterPhyId, topicName));
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(e.getResultStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<String, String>> getTopicConfigFromKafka(Long clusterPhyId, String topicName) {
|
||||
Result<List<KafkaConfigDetail>> configResult = this.getTopicConfigDetailFromKafka(clusterPhyId, topicName);
|
||||
if (configResult.failed()) {
|
||||
return Result.buildFromIgnoreData(configResult);
|
||||
}
|
||||
|
||||
return Result.buildSuc(configResult.getData().stream().filter(elem -> elem.getValue() != null).collect(Collectors.toMap(KafkaConfigDetail::getName, KafkaConfigDetail::getValue)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Properties> getConfigNamesAndDocs(Long clusterPhyId) {
|
||||
ClusterPhy clusterPhy = clusterPhyService.getClusterByCluster(clusterPhyId);
|
||||
if (clusterPhy == null) {
|
||||
// 默认
|
||||
return TOPIC_CONFIG_NAMES_AND_DOCS.get(0).v3();
|
||||
}
|
||||
|
||||
Long clusterVersionL = VersionUtil.normailze(clusterPhy.getKafkaVersion());
|
||||
for (Triple<Long, Long, List<Properties>> elem: TOPIC_CONFIG_NAMES_AND_DOCS) {
|
||||
if (elem.v1() <= clusterVersionL && clusterVersionL <= elem.v2()) {
|
||||
return elem.v3();
|
||||
}
|
||||
}
|
||||
|
||||
return TOPIC_CONFIG_NAMES_AND_DOCS.get(0).v3();
|
||||
}
|
||||
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
|
||||
private Result<List<KafkaConfigDetail>> getTopicConfigByZKClient(VersionItemParam itemParam) {
|
||||
TopicParam param = (TopicParam) itemParam;
|
||||
|
||||
Result<Properties> propertiesResult = this.getTopicConfigByZKClient(param.getClusterPhyId(), param.getTopicName());
|
||||
if (propertiesResult.failed()) {
|
||||
return Result.buildFromIgnoreData(propertiesResult);
|
||||
}
|
||||
|
||||
return Result.buildSuc(KafkaConfigConverter.convert2KafkaTopicConfigDetailList(
|
||||
this.getConfigNamesAndDocs(param.getClusterPhyId()),
|
||||
propertiesResult.getData()
|
||||
));
|
||||
}
|
||||
|
||||
private Result<Properties> getTopicConfigByZKClient(Long clusterPhyId, String topicName) {
|
||||
try {
|
||||
Topic topic = topicService.getTopic(clusterPhyId, topicName);
|
||||
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(clusterPhyId);
|
||||
|
||||
Properties properties = kafkaZkClient.getEntityConfigs("topics", topic.getTopicName());
|
||||
for (Object key: properties.keySet()) {
|
||||
properties.getProperty((String) key);
|
||||
}
|
||||
|
||||
return Result.buildSuc(properties);
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(clusterPhyId, topicName));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getTopicConfigByZKClient||clusterPhyId={}||topicName={}||errMsg=exception", clusterPhyId, topicName, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<KafkaConfigDetail>> getTopicConfigByKafkaClient(VersionItemParam itemParam) {
|
||||
TopicParam param = (TopicParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(param.getClusterPhyId());
|
||||
|
||||
Topic metadata = topicService.getTopic(param.getClusterPhyId(), param.getTopicName());
|
||||
|
||||
ConfigResource configResource = new ConfigResource(ConfigResource.Type.TOPIC, metadata.getTopicName());
|
||||
DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(
|
||||
Arrays.asList(configResource),
|
||||
buildDescribeConfigsOptions()
|
||||
);
|
||||
|
||||
Map<ConfigResource, Config> configMap = describeConfigsResult.all().get();
|
||||
|
||||
return Result.buildSuc(
|
||||
KafkaConfigConverter.convert2KafkaConfigDetailList(configMap.get(configResource))
|
||||
);
|
||||
} catch (NotExistException nee) {
|
||||
return Result.buildFromRSAndMsg(ResultStatus.NOT_EXIST, MsgConstant.getTopicNotExist(param.getClusterPhyId(), param.getTopicName()));
|
||||
} catch (Exception e) {
|
||||
log.error("method=getTopicConfigByKafkaClient||param={}||errMsg=exception!", param, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private DescribeConfigsOptions buildDescribeConfigsOptions() {
|
||||
return new DescribeConfigsOptions().includeDocumentation(false).includeSynonyms(false).timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS);
|
||||
}
|
||||
|
||||
private Result<Void> modifyTopicConfigByKafkaClient(VersionItemParam itemParam) {
|
||||
KafkaTopicConfigParam kafkaTopicConfigParam = (KafkaTopicConfigParam) itemParam;
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(kafkaTopicConfigParam.getClusterPhyId());
|
||||
|
||||
List<AlterConfigOp> configOpList = new ArrayList<>();
|
||||
for (Map.Entry<String, String> changedConfig: kafkaTopicConfigParam.getChangedProps().entrySet()) {
|
||||
configOpList.add(new AlterConfigOp(new ConfigEntry(changedConfig.getKey(), changedConfig.getValue()), AlterConfigOp.OpType.SET));
|
||||
}
|
||||
|
||||
Map<ConfigResource, Collection<AlterConfigOp>> configMap = new HashMap<>();
|
||||
configMap.put(new ConfigResource(ConfigResource.Type.TOPIC, kafkaTopicConfigParam.getTopicName()), configOpList);
|
||||
|
||||
AlterConfigsResult alterConfigsResult = adminClient.incrementalAlterConfigs(configMap, new AlterConfigsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
alterConfigsResult.all().get();
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyTopicConfigByKafkaClient||param={}||errMsg={}", kafkaTopicConfigParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> modifyTopicConfigByZKClientAndNodeVersionV1(VersionItemParam itemParam) {
|
||||
KafkaTopicConfigParam kafkaTopicConfigParam = (KafkaTopicConfigParam) itemParam;
|
||||
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(kafkaTopicConfigParam.getClusterPhyId());
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(kafkaTopicConfigParam.getClusterPhyId());
|
||||
|
||||
Properties allProps = new Properties();
|
||||
|
||||
// 历史配置 + 当前修改的配置 -》获取到最终的配置
|
||||
allProps.putAll(kafkaZkClient.getEntityConfigs(ConfigType.Topic(), kafkaTopicConfigParam.getTopicName()));
|
||||
allProps.putAll(kafkaTopicConfigParam.getChangedProps());
|
||||
allProps.putAll(kafkaZkClient.getEntityConfigs(ConfigType.Topic(), kafkaTopicConfigParam.getTopicName()));
|
||||
|
||||
// 检查参数是否合法
|
||||
adminZkClient.validateTopicConfig(kafkaTopicConfigParam.getTopicName(), allProps);
|
||||
|
||||
// 修改配置的数据节点
|
||||
kafkaZkClient.setOrCreateEntityConfigs(ConfigType.Topic(), kafkaTopicConfigParam.getTopicName(), allProps);
|
||||
|
||||
// 修改配置的通知节点
|
||||
kafkaZKDAO.createConfigChangeNotificationVersionOne(kafkaTopicConfigParam.getClusterPhyId(), ConfigType.Topic(), kafkaTopicConfigParam.getTopicName());
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyTopicConfigByZKClientAndNodeVersionV1||param={}||errMsg=exception!", kafkaTopicConfigParam, e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
|
||||
return Result.buildSuc();
|
||||
}
|
||||
|
||||
private Result<Void> modifyTopicConfigByZKClientAndNodeVersionV2(VersionItemParam itemParam) {
|
||||
KafkaTopicConfigParam kafkaTopicConfigParam = (KafkaTopicConfigParam) itemParam;
|
||||
try {
|
||||
AdminZkClient adminZkClient = kafkaAdminZKClient.getKafkaZKWrapClient(kafkaTopicConfigParam.getClusterPhyId());
|
||||
KafkaZkClient kafkaZkClient = kafkaAdminZKClient.getClient(kafkaTopicConfigParam.getClusterPhyId());
|
||||
|
||||
// 获取ZK配置
|
||||
Properties properties = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), kafkaTopicConfigParam.getTopicName());
|
||||
properties.putAll(kafkaTopicConfigParam.getChangedProps());
|
||||
|
||||
// 修改配置
|
||||
adminZkClient.changeTopicConfig(kafkaTopicConfigParam.getTopicName(), properties);
|
||||
} catch (Exception e) {
|
||||
log.error("method=modifyTopicConfigByZKClient||param={}||errMsg={}", kafkaTopicConfigParam, e.getMessage());
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.ZK_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
return Result.buildSuc();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic.impl;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.google.common.collect.Table;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.dto.metrices.MetricsTopicDTO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.broker.Broker;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.PartitionMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.TopicMetrics;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.metric.TopicMetricParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.PaginationResult;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.*;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.TopicMetricPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.MsgConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.jmx.JmxConnectorWrap;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.BeanUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.cache.CollectMetricsLocalCache;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.broker.BrokerService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.health.score.HealthScoreService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.partition.PartitionMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.BaseMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.cache.LoadedClusterPhyCache;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.es.dao.TopicMetricESDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaJMXClient;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.management.InstanceNotFoundException;
|
||||
import javax.management.ObjectName;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.version.metrics.PartitionMetricVersionItems.PARTITION_METRIC_MESSAGES;
|
||||
|
||||
/**
|
||||
*/
|
||||
@Service
|
||||
public class TopicMetricServiceImpl extends BaseMetricService implements TopicMetricService {
|
||||
|
||||
private static final ILog LOGGER = LogFactory.getLog( TopicMetricServiceImpl.class);
|
||||
|
||||
public static final String TOPIC_METHOD_DO_NOTHING = "doNothing";
|
||||
public static final String TOPIC_METHOD_GET_HEALTH_SCORE = "getMetricHealthScore";
|
||||
public static final String TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX = "getMetricFromKafkaByTotalBrokerJmx";
|
||||
public static final String TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_PARTITION_OF_BROKER_JMX = "getMetricFromKafkaByTotalPartitionOfBrokerJmx";
|
||||
public static final String TOPIC_METHOD_GET_MESSAGES = "getMessages";
|
||||
public static final String TOPIC_METHOD_GET_REPLICAS_COUNT = "getReplicasCount";
|
||||
|
||||
@Autowired
|
||||
private HealthScoreService healthScoreService;
|
||||
|
||||
@Autowired
|
||||
private KafkaJMXClient kafkaJMXClient;
|
||||
|
||||
@Autowired
|
||||
private BrokerService brokerService;
|
||||
|
||||
@Autowired
|
||||
private TopicService topicService;
|
||||
|
||||
@Autowired
|
||||
private PartitionMetricService partitionMetricService;
|
||||
|
||||
@Autowired
|
||||
private TopicMetricESDAO topicMetricESDAO;
|
||||
|
||||
private final Cache<Long, Map<String, TopicMetrics>> topicLatestMetricsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(5, TimeUnit.MINUTES)
|
||||
.maximumSize(200)
|
||||
.build();
|
||||
|
||||
@Scheduled(cron = "0 0/2 * * * ?")
|
||||
private void flushClusterLatestMetricsCache() {
|
||||
for (ClusterPhy clusterPhy: LoadedClusterPhyCache.listAll().values()) {
|
||||
this.updateCacheAndGetMetrics(clusterPhy.getId());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected VersionItemTypeEnum getVersionItemType() {
|
||||
return VersionItemTypeEnum.METRIC_TOPIC;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<String> listMetricPOFields(){
|
||||
return BeanUtil.listBeanFields(TopicMetricPO.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initRegisterVCHandler(){
|
||||
registerVCHandler( TOPIC_METHOD_DO_NOTHING, this::doNothing);
|
||||
registerVCHandler( TOPIC_METHOD_GET_HEALTH_SCORE, this::getMetricHealthScore);
|
||||
registerVCHandler( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX, this::getMetricFromKafkaByTotalBrokerJmx);
|
||||
registerVCHandler( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_PARTITION_OF_BROKER_JMX, this::getMetricFromKafkaByTotalPartitionOfBrokerJmx );
|
||||
registerVCHandler( TOPIC_METHOD_GET_REPLICAS_COUNT, this::getReplicasCount);
|
||||
registerVCHandler( TOPIC_METHOD_GET_MESSAGES, this::getMessages);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<TopicMetrics>> collectTopicMetricsFromKafkaWithCacheFirst(Long clusterPhyId, String topicName, String metricName) {
|
||||
List<TopicMetrics> metricsList = CollectMetricsLocalCache.getTopicMetrics(clusterPhyId, topicName, metricName);
|
||||
if(null != metricsList) {
|
||||
return Result.buildSuc(metricsList);
|
||||
}
|
||||
|
||||
Result<List<TopicMetrics>> metricsResult = this.collectTopicMetricsFromKafka(clusterPhyId, topicName, metricName);
|
||||
if(null == metricsResult || metricsResult.failed() || null == metricsResult.getData() || metricsResult.getData().isEmpty()) {
|
||||
return metricsResult;
|
||||
}
|
||||
|
||||
// 更新cache
|
||||
TopicMetrics metrics = metricsResult.getData().get(0);
|
||||
metrics.getMetrics().entrySet().forEach(
|
||||
metricEntry -> CollectMetricsLocalCache.putTopicMetrics(
|
||||
clusterPhyId,
|
||||
metrics.getTopic(),
|
||||
metricEntry.getKey(),
|
||||
metricsResult.getData()
|
||||
)
|
||||
);
|
||||
|
||||
return metricsResult;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<TopicMetrics>> collectTopicMetricsFromKafka(Long clusterId, String topic, String metric){
|
||||
try {
|
||||
TopicMetricParam topicMetricParam = new TopicMetricParam(clusterId, topic, metric);
|
||||
return (Result<List<TopicMetrics>>)doVCHandler(clusterId, metric, topicMetricParam);
|
||||
} catch (VCHandlerNotExistException e) {
|
||||
return Result.buildFailure(VC_HANDLE_NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, TopicMetrics> getLatestMetricsFromCacheFirst(Long clusterPhyId) {
|
||||
Map<String, TopicMetrics> metricsMap = topicLatestMetricsCache.getIfPresent(clusterPhyId);
|
||||
if (metricsMap != null) {
|
||||
return metricsMap;
|
||||
}
|
||||
|
||||
return this.updateCacheAndGetMetrics(clusterPhyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TopicMetrics getTopicLatestMetricsFromES(Long clusterPhyId, Integer brokerId, String topicName, List<String> metricNames) {
|
||||
TopicMetricPO topicMetricPO = topicMetricESDAO.getTopicLatestMetricByBrokerId(clusterPhyId,
|
||||
topicName, brokerId, metricNames);
|
||||
|
||||
return ConvertUtil.obj2Obj(topicMetricPO, TopicMetrics.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TopicMetrics> listTopicLatestMetricsFromES(Long clusterPhyId, List<String> topicNames, List<String> metricNames) {
|
||||
List<TopicMetricPO> topicMetricPOs = topicMetricESDAO.listTopicLatestMetric(clusterPhyId, topicNames, metricNames);
|
||||
|
||||
return ConvertUtil.list2List(topicMetricPOs, TopicMetrics.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TopicMetrics getTopicLatestMetricsFromES(Long clusterPhyId, String topicName, List<String> metricNames) {
|
||||
TopicMetricPO topicMetricPO = topicMetricESDAO.getTopicLatestMetric(clusterPhyId, topicName, metricNames);
|
||||
|
||||
return ConvertUtil.obj2Obj(topicMetricPO, TopicMetrics.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricMultiLinesVO>> listTopicMetricsFromES(Long clusterId, MetricsTopicDTO dto) {
|
||||
Long startTime = dto.getStartTime();
|
||||
Long endTime = dto.getEndTime();
|
||||
Integer topN = dto.getTopNu();
|
||||
String aggType = dto.getAggType();
|
||||
List<String> topics = dto.getTopics();
|
||||
List<String> metrics = dto.getMetricsNames();
|
||||
|
||||
Table<String/*metric*/, String/*topics*/, List<MetricPointVO>> retTable;
|
||||
if(CollectionUtils.isEmpty(topics)) {
|
||||
//如果 es 中获取不到topN的topic就使用从数据库中获取的topics
|
||||
List<String> defaultTopics = listTopNTopics(clusterId, topN);
|
||||
retTable = topicMetricESDAO.listTopicMetricsByTopN(clusterId, defaultTopics, metrics, aggType, topN, startTime, endTime );
|
||||
}else {
|
||||
retTable = topicMetricESDAO.listTopicMetricsByTopics(clusterId, metrics, aggType, topics, startTime, endTime);
|
||||
}
|
||||
|
||||
List<MetricMultiLinesVO> multiLinesVOS = metricMap2VO(clusterId, retTable.rowMap());
|
||||
return Result.buildSuc(multiLinesVOS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<TopicMetrics>> listTopicMaxMinMetrics(Long clusterPhyId, List<String> topics, String metric,
|
||||
boolean max, Long startTime, Long endTime){
|
||||
List<TopicMetricPO> ret = topicMetricESDAO.listTopicMaxMinMetrics(clusterPhyId, topics, metric, max, startTime, endTime);
|
||||
return Result.buildSuc(ConvertUtil.list2List(ret, TopicMetrics.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<String, MetricPointVO>> getAggMetricPointFromES(Long clusterPhyId,
|
||||
List<String> topicNames,
|
||||
String metricName,
|
||||
AggTypeEnum aggTypeEnum,
|
||||
Long startTime,
|
||||
Long endTime) {
|
||||
if (!(AggTypeEnum.MAX.equals(aggTypeEnum) || AggTypeEnum.MIN.equals(aggTypeEnum))) {
|
||||
// 除了max和min方式之外的聚合方式
|
||||
Table<String/*topics*/, String/*metric*/, MetricPointVO> metricPointTable = topicMetricESDAO.getTopicsAggsMetricsValue(clusterPhyId, topicNames,
|
||||
Arrays.asList(metricName), aggTypeEnum.getAggType(), startTime, endTime);
|
||||
|
||||
Map<String/*Topic名称*/, MetricPointVO> voMap = new HashMap<>();
|
||||
for(String topic : metricPointTable.rowKeySet()){
|
||||
List<MetricPointVO> metricPoints = new ArrayList<>(metricPointTable.values()).stream().filter(elem -> elem.getName().equals(metricName)).collect(Collectors.toList());
|
||||
if (ValidateUtils.isEmptyList(metricPoints)) {
|
||||
continue;
|
||||
}
|
||||
voMap.put(topic, metricPoints.get(0));
|
||||
}
|
||||
|
||||
return Result.buildSuc(voMap);
|
||||
}
|
||||
|
||||
// max和min方式之外的聚合方式
|
||||
Result<List<TopicMetrics>> metricListResult = this.listTopicMaxMinMetrics(clusterPhyId, topicNames, metricName, AggTypeEnum.MAX.equals(aggTypeEnum), startTime, endTime);
|
||||
if (metricListResult.failed()) {
|
||||
return Result.buildFromIgnoreData(metricListResult);
|
||||
}
|
||||
|
||||
Map<String, MetricPointVO> voMap = new HashMap<>();
|
||||
for (TopicMetrics metrics: metricListResult.getData()) {
|
||||
Float value = metrics.getMetric(metricName);
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
voMap.put(metrics.getTopic(), new MetricPointVO(metricName, metrics.getTimestamp(), String.valueOf(value), aggTypeEnum.getAggType()));
|
||||
}
|
||||
|
||||
return Result.buildSuc(voMap);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<List<MetricPointVO>> getMetricPointsFromES(Long clusterId, String topicName, MetricDTO dto) {
|
||||
Table<String/*topics*/, String/*metric*/, MetricPointVO> metricPointTable = topicMetricESDAO.getTopicsAggsMetricsValue(clusterId, Arrays.asList(topicName),
|
||||
dto.getMetricsNames(), dto.getAggType(), dto.getStartTime(), dto.getEndTime());
|
||||
|
||||
List<MetricPointVO> metricPoints = new ArrayList<>(metricPointTable.values());
|
||||
return Result.buildSuc(metricPoints);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Map<String/*Topic*/, List<MetricPointVO>>> getMetricPointsFromES(Long clusterId, List<String> topicNames, MetricDTO dto) {
|
||||
Table<String/*topics*/, String/*metric*/, MetricPointVO> metricPointTable = topicMetricESDAO.getTopicsAggsMetricsValue(clusterId, topicNames,
|
||||
dto.getMetricsNames(), dto.getAggType(), dto.getStartTime(), dto.getEndTime());
|
||||
|
||||
Map<String/*Topic名称*/, List<MetricPointVO>> retMap = new HashMap<>();
|
||||
for(String topic : metricPointTable.rowKeySet()){
|
||||
retMap.put(topic, new ArrayList<>(metricPointTable.row(topic).values()));
|
||||
}
|
||||
|
||||
return Result.buildSuc(retMap);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PaginationResult<TopicMetrics> pagingTopicWithLatestMetricsFromES(Long clusterId, List<String> metricNameList,
|
||||
SearchSort sort, SearchFuzzy fuzzy,
|
||||
List<SearchShould> shoulds, List<SearchTerm> terms,
|
||||
SearchPage page){
|
||||
setQueryMetricFlag(sort);
|
||||
setQueryMetricFlag(fuzzy);
|
||||
setQueryMetricFlag(terms);
|
||||
setQueryMetricFlag(shoulds);
|
||||
|
||||
List<TopicMetricPO> topicMetricPOS = topicMetricESDAO.listTopicWithLatestMetrics(clusterId, sort, fuzzy, shoulds, terms);
|
||||
|
||||
int startIdx = Math.min((page.getPageNo() - 1) * page.getPageSize(), topicMetricPOS.size());
|
||||
int endIdx = Math.min(startIdx +page.getPageSize(), topicMetricPOS.size());
|
||||
|
||||
List<TopicMetricPO> subList = topicMetricPOS.subList(startIdx, endIdx);
|
||||
|
||||
return PaginationResult.buildSuc(ConvertUtil.list2List(subList, TopicMetrics.class), topicMetricPOS.size(), page.getPageNo(), page.getPageSize());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Integer> countMetricValueOccurrencesFromES(Long clusterPhyId, String topicName,
|
||||
SearchTerm searchMatch, Long startTime, Long endTime) {
|
||||
setQueryMetricFlag(searchMatch);
|
||||
int count = topicMetricESDAO.countMetricValue(clusterPhyId, topicName, searchMatch, startTime, endTime);
|
||||
if(count < 0){
|
||||
return Result.buildFail();
|
||||
}
|
||||
|
||||
return Result.buildSuc(count);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
private Map<String, TopicMetrics> updateCacheAndGetMetrics(Long clusterPhyId) {
|
||||
List<String> topicNames = topicService.listTopicsFromDB(clusterPhyId)
|
||||
.stream().map(Topic::getTopicName).collect(Collectors.toList());
|
||||
|
||||
List<TopicMetrics> metrics = listTopicLatestMetricsFromES(clusterPhyId, topicNames, Arrays.asList());
|
||||
|
||||
Map<String, TopicMetrics> metricsMap = metrics.stream()
|
||||
.collect(Collectors.toMap(TopicMetrics::getTopic, Function.identity()));
|
||||
|
||||
topicLatestMetricsCache.put(clusterPhyId, metricsMap);
|
||||
return metricsMap;
|
||||
}
|
||||
|
||||
|
||||
private List<String> listTopNTopics(Long clusterId, int topN){
|
||||
List<Topic> topics = topicService.listTopicsFromDB(clusterId);
|
||||
if(CollectionUtils.isEmpty(topics)){return new ArrayList<>();}
|
||||
|
||||
return topics.subList(0, Math.min(topN, topics.size())).stream().map(Topic::getTopicName).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private Result<List<TopicMetrics>> doNothing(VersionItemParam metricParam) {
|
||||
TopicMetricParam topicMetricParam = (TopicMetricParam)metricParam;
|
||||
|
||||
return Result.buildSuc(Arrays.asList(new TopicMetrics(topicMetricParam.getTopic(), topicMetricParam.getClusterId(), true)));
|
||||
}
|
||||
|
||||
private Result<List<TopicMetrics>> getMessages(VersionItemParam param) {
|
||||
TopicMetricParam topicMetricParam = (TopicMetricParam)param;
|
||||
|
||||
String metric = topicMetricParam.getMetric();
|
||||
String topic = topicMetricParam.getTopic();
|
||||
Long clusterId = topicMetricParam.getClusterId();
|
||||
|
||||
Result<List<PartitionMetrics>> metricsResult = partitionMetricService.collectPartitionsMetricsFromKafkaWithCache(clusterId, topic, PARTITION_METRIC_MESSAGES);
|
||||
if (!metricsResult.hasData()) {
|
||||
return Result.buildFromIgnoreData(metricsResult);
|
||||
}
|
||||
|
||||
Float sumMessages = 0.0f;
|
||||
for(PartitionMetrics metrics : metricsResult.getData()) {
|
||||
Float messages = metrics.getMetric(PARTITION_METRIC_MESSAGES);
|
||||
if (messages == null) {
|
||||
return Result.buildFromRSAndMsg(KAFKA_OPERATE_FAILED, MsgConstant.getPartitionNotExist(clusterId, topic, metrics.getPartitionId()));
|
||||
}
|
||||
|
||||
sumMessages += messages;
|
||||
}
|
||||
|
||||
TopicMetrics metrics = new TopicMetrics(topic, clusterId, true);
|
||||
metrics.putMetric(metric, sumMessages);
|
||||
return Result.buildSuc(Arrays.asList(metrics));
|
||||
}
|
||||
|
||||
private Result<List<TopicMetrics>> getReplicasCount(VersionItemParam param) {
|
||||
TopicMetricParam topicMetricParam = (TopicMetricParam)param;
|
||||
|
||||
String metric = topicMetricParam.getMetric();
|
||||
String topicName = topicMetricParam.getTopic();
|
||||
Long clusterId = topicMetricParam.getClusterId();
|
||||
|
||||
Topic topic = topicService.getTopicFromCacheFirst(clusterId, topicName);
|
||||
if (topic == null) {
|
||||
return Result.buildFailure(TOPIC_NOT_EXIST);
|
||||
}
|
||||
|
||||
Integer replicasCount = topic.getReplicaNum() * topic.getPartitionNum();
|
||||
|
||||
TopicMetrics topicMetric = new TopicMetrics(topicName, clusterId, true);
|
||||
topicMetric.putMetric(metric, replicasCount.floatValue());
|
||||
|
||||
return Result.buildSuc(Arrays.asList(topicMetric));
|
||||
}
|
||||
|
||||
/**
|
||||
* 回去 topic 健康分指标
|
||||
* @param param
|
||||
* @return
|
||||
*/
|
||||
private Result<List<TopicMetrics>> getMetricHealthScore(VersionItemParam param) {
|
||||
TopicMetricParam topicMetricParam = (TopicMetricParam)param;
|
||||
|
||||
String topic = topicMetricParam.getTopic();
|
||||
Long clusterId = topicMetricParam.getClusterId();
|
||||
|
||||
TopicMetrics topicMetric = healthScoreService.calTopicHealthScore(clusterId, topic);
|
||||
return Result.buildSuc(Arrays.asList(topicMetric));
|
||||
}
|
||||
|
||||
/**
|
||||
* 从JMX中获取LogSize
|
||||
*/
|
||||
private Result<List<TopicMetrics>> getMetricFromKafkaByTotalPartitionOfBrokerJmx(VersionItemParam param) {
|
||||
TopicMetricParam topicMetricParam = (TopicMetricParam)param;
|
||||
|
||||
String metric = topicMetricParam.getMetric();
|
||||
String topicName = topicMetricParam.getTopic();
|
||||
Long clusterId = topicMetricParam.getClusterId();
|
||||
|
||||
//1、获取topic所在的broker以及partition分布信息
|
||||
Topic topic = topicService.getTopicFromCacheFirst(clusterId, topicName);
|
||||
Map<Integer, List<Integer>> broker2PartitionMap = new HashMap<>();
|
||||
topic.getPartitionMap().entrySet().stream().forEach(entry -> {
|
||||
for (Integer brokerId: entry.getValue()) {
|
||||
broker2PartitionMap.putIfAbsent(brokerId, new ArrayList<>());
|
||||
broker2PartitionMap.get(brokerId).add(entry.getKey());
|
||||
}
|
||||
});
|
||||
|
||||
//2、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
//3、获取broker列表的jmx连接，开始采集指标
|
||||
float topicLogSize = 0f;
|
||||
List<TopicMetrics> topicMetricsList = new ArrayList<>();
|
||||
for(Map.Entry<Integer, List<Integer>> brokerAndPartitionEntry : broker2PartitionMap.entrySet()){
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, brokerAndPartitionEntry.getKey());
|
||||
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){
|
||||
return Result.buildFailure(VC_JMX_CONNECT_ERROR);
|
||||
}
|
||||
|
||||
float brokerTopicLogSize = 0f;
|
||||
for(Integer partitionId : brokerAndPartitionEntry.getValue()){
|
||||
try {
|
||||
String value = jmxConnectorWrap.getAttribute(new ObjectName(jmxInfo.getJmxObjectName()
|
||||
+ ",topic=" + topicName + ",partition=" + partitionId),
|
||||
jmxInfo.getJmxAttribute()).toString();
|
||||
|
||||
brokerTopicLogSize += Double.valueOf(value);
|
||||
} catch (InstanceNotFoundException e) {
|
||||
// ignore
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("getMetricFromKafkaByTotalPartitionOfBrokerJmx||cluster={}||brokerId={}||topic={}||metrics={}||jmx={}||msg={}",
|
||||
clusterId, brokerAndPartitionEntry.getKey(), topicName, metric, jmxInfo.getJmxObjectName(), e.getClass().getName());
|
||||
}
|
||||
}
|
||||
|
||||
topicLogSize += brokerTopicLogSize;
|
||||
TopicMetrics topicBrokerMetric = new TopicMetrics(topicName, clusterId, brokerAndPartitionEntry.getKey(), false);
|
||||
topicBrokerMetric.putMetric(metric, brokerTopicLogSize);
|
||||
topicMetricsList.add(topicBrokerMetric);
|
||||
}
|
||||
|
||||
TopicMetrics topicMetrics = new TopicMetrics(topicName, clusterId, true);
|
||||
topicMetrics.putMetric(metric, topicLogSize);
|
||||
topicMetricsList.add(topicMetrics);
|
||||
|
||||
return Result.buildSuc(topicMetricsList);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从JMX中获取指标
|
||||
*/
|
||||
private Result<List<TopicMetrics>> getMetricFromKafkaByTotalBrokerJmx(VersionItemParam param){
|
||||
TopicMetricParam topicMetricParam = (TopicMetricParam)param;
|
||||
|
||||
String metric = topicMetricParam.getMetric();
|
||||
String topic = topicMetricParam.getTopic();
|
||||
Long clusterId = topicMetricParam.getClusterId();
|
||||
|
||||
//1、获取jmx的属性信息
|
||||
VersionJmxInfo jmxInfo = getJMXInfo(clusterId, metric);
|
||||
if(null == jmxInfo){return Result.buildFailure(VC_ITEM_JMX_NOT_EXIST);}
|
||||
|
||||
//2、获取topic所在的broker信息
|
||||
List<Broker> brokers = this.listAliveBrokersByTopic(clusterId, topic);
|
||||
if(CollectionUtils.isEmpty(brokers)){return Result.buildFailure(BROKER_NOT_EXIST);}
|
||||
|
||||
//3、获取jmx连接，开始采集指标
|
||||
float topicMetricValue = 0f;
|
||||
List<TopicMetrics> topicMetrics = new ArrayList<>();
|
||||
for(Broker broker : brokers){
|
||||
Integer brokerId = broker.getBrokerId();
|
||||
JmxConnectorWrap jmxConnectorWrap = kafkaJMXClient.getClientWithCheck(clusterId, brokerId);
|
||||
|
||||
if (ValidateUtils.isNull(jmxConnectorWrap)){return Result.buildFailure(VC_JMX_INIT_ERROR);}
|
||||
|
||||
try {
|
||||
String value = jmxConnectorWrap.getAttribute(new ObjectName(jmxInfo.getJmxObjectName() + ",topic=" + topic),
|
||||
jmxInfo.getJmxAttribute()).toString();
|
||||
|
||||
topicMetricValue += Float.valueOf(value);
|
||||
|
||||
TopicMetrics topicBrokerMetric = new TopicMetrics(topic, clusterId, brokerId, false);
|
||||
topicBrokerMetric.putMetric(metric, Float.valueOf(value));
|
||||
|
||||
topicMetrics.add(topicBrokerMetric);
|
||||
} catch (InstanceNotFoundException e) {
|
||||
// ignore
|
||||
} catch (Exception e) {
|
||||
LOGGER.error("method=getMetricFromKafkaByTotalBrokerJmx||cluster={}||brokerId={}||topic={}||metrics={}||jmx={}||msg={}",
|
||||
clusterId, brokerId, topic, metric, jmxInfo.getJmxObjectName(), e.getClass().getName());
|
||||
}
|
||||
}
|
||||
|
||||
TopicMetrics topicMetric = new TopicMetrics(topic, clusterId, true);
|
||||
topicMetric.putMetric(metric, topicMetricValue);
|
||||
topicMetrics.add(topicMetric);
|
||||
|
||||
return Result.buildSuc(topicMetrics);
|
||||
}
|
||||
|
||||
private List<Broker> listAliveBrokersByTopic(Long clusterPhyId, String topicName) {
|
||||
List<Broker> aliveBrokerList = brokerService.listAliveBrokersFromCacheFirst(clusterPhyId);
|
||||
|
||||
Topic topic = topicService.getTopicFromCacheFirst(clusterPhyId, topicName);
|
||||
if (topic == null) {
|
||||
return aliveBrokerList;
|
||||
}
|
||||
|
||||
return aliveBrokerList.stream().filter(elem -> topic.getBrokerIdSet().contains(elem.getBrokerId())).collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.topic.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.cluster.ClusterPhy;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.Result;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.result.ResultStatus;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.Topic;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicConfig;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.po.topic.TopicPO;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.KafkaConstant;
|
||||
import com.xiaojukeji.know.streaming.km.common.converter.TopicConverter;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.cluster.ClusterRunStateEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.AdminOperateException;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.NotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ConvertUtil;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.DateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.topic.TopicService;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.kafka.KafkaAdminClient;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.mysql.topic.TopicDAO;
|
||||
import com.xiaojukeji.know.streaming.km.persistence.zk.KafkaZKDAO;
|
||||
import kafka.zk.TopicsZNode;
|
||||
import org.apache.kafka.clients.admin.*;
|
||||
import org.apache.kafka.common.TopicPartitionInfo;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Service
|
||||
public class TopicServiceImpl implements TopicService {
|
||||
private static final ILog log = LogFactory.getLog(TopicConfigServiceImpl.class);
|
||||
|
||||
@Autowired
|
||||
private TopicDAO topicDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaZKDAO kafkaZKDAO;
|
||||
|
||||
@Autowired
|
||||
private KafkaAdminClient kafkaAdminClient;
|
||||
|
||||
private final Cache<Long, Map<String, Topic>> topicsCache = Caffeine.newBuilder()
|
||||
.expireAfterWrite(90, TimeUnit.SECONDS)
|
||||
.maximumSize(1000)
|
||||
.build();
|
||||
|
||||
@Override
|
||||
public Result<List<Topic>> listTopicsFromKafka(ClusterPhy clusterPhy) {
|
||||
if (clusterPhy.getRunState().equals(ClusterRunStateEnum.RUN_ZK.getRunState())) {
|
||||
return this.getTopicsFromZKClient(clusterPhy);
|
||||
}
|
||||
|
||||
return this.getTopicsFromAdminClient(clusterPhy);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<Integer, List<Integer>> getTopicPartitionMapFromKafka(Long clusterPhyId, String topicName) throws NotExistException, AdminOperateException {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhyId);
|
||||
|
||||
try {
|
||||
DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(
|
||||
Arrays.asList(topicName),
|
||||
new DescribeTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS)
|
||||
);
|
||||
|
||||
TopicDescription description = describeTopicsResult.all().get().get(topicName);
|
||||
|
||||
Map<Integer, List<Integer>> partitionMap = new HashMap<>();
|
||||
for (TopicPartitionInfo partitionInfo: description.partitions()) {
|
||||
partitionMap.put(partitionInfo.partition(), partitionInfo.replicas().stream().map(elem -> elem.id()).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
return partitionMap;
|
||||
} catch (Exception e) {
|
||||
log.error("method=getTopicPartitionMapFromKafka||clusterPhyId={}||topicName={}||errMsg=exception", clusterPhyId, topicName, e);
|
||||
throw new AdminOperateException("get topic info from kafka failed", e, ResultStatus.KAFKA_OPERATE_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Topic> listTopicsFromDB(Long clusterPhyId) {
|
||||
return TopicConverter.convert2TopicList(this.getTopicsFromDB(clusterPhyId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Topic> listTopicsFromCacheFirst(Long clusterPhyId) {
|
||||
Map<String, Topic> topicMap = topicsCache.getIfPresent(clusterPhyId);
|
||||
if (topicMap == null) {
|
||||
topicMap = this.getTopicsAndUpdateCache(clusterPhyId);
|
||||
}
|
||||
|
||||
return topicMap == null? new ArrayList<>(): new ArrayList<>(topicMap.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listRecentUpdateTopicNamesFromDB(Long clusterPhyId, Integer time) {
|
||||
Date updateTime = DateUtils.getBeforeSeconds(new Date(), time);
|
||||
LambdaQueryWrapper<TopicPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(TopicPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.ge(TopicPO::getUpdateTime, updateTime);
|
||||
List<TopicPO> topicPOS = topicDAO.selectList(lambdaQueryWrapper);
|
||||
if (topicPOS.isEmpty()){
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return topicPOS.stream().map(TopicPO::getTopicName).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getTopicSizeFromCacheFirst(Long clusterPhyId) {
|
||||
List<Topic> topicList = this.listTopicsFromCacheFirst(clusterPhyId);
|
||||
return topicList == null? 0: topicList.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer getReplicaSizeFromCacheFirst(Long clusterPhyId) {
|
||||
List<Topic> topicList = this.listTopicsFromCacheFirst(clusterPhyId);
|
||||
if (ValidateUtils.isEmptyList(topicList)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return topicList.stream()
|
||||
.map(elem -> elem.getPartitionNum() * elem.getReplicaNum())
|
||||
.reduce(Integer::sum)
|
||||
.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Topic getTopic(Long clusterPhyId, String topicName) {
|
||||
TopicPO po = this.getTopicFromDB(clusterPhyId, topicName);
|
||||
if (po == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return TopicConverter.convert2Topic(po);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Topic getTopicFromCacheFirst(Long clusterPhyId, String topicName) {
|
||||
Map<String, Topic> topicMap = topicsCache.getIfPresent(clusterPhyId);
|
||||
if (topicMap == null) {
|
||||
topicMap = this.getTopicsAndUpdateCache(clusterPhyId);
|
||||
}
|
||||
|
||||
return topicMap == null? null: topicMap.get(topicName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int addNewTopic2DB(TopicPO po) {
|
||||
return topicDAO.replaceAll(po);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int deleteTopicInDB(Long clusterPhyId, String topicName) {
|
||||
LambdaQueryWrapper<TopicPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(TopicPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(TopicPO::getTopicName, topicName);
|
||||
return topicDAO.delete(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchReplaceMetadata(Long clusterPhyId, List<Topic> presentTopicList) {
|
||||
Map<String, Topic> presentTopicMap = presentTopicList.stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity()));
|
||||
|
||||
List<TopicPO> dbTopicPOList = this.getTopicsFromDB(clusterPhyId);
|
||||
|
||||
// 新旧合并
|
||||
for (TopicPO dbTopicPO: dbTopicPOList) {
|
||||
Topic topic = presentTopicMap.remove(dbTopicPO.getTopicName());
|
||||
if (topic == null) {
|
||||
topicDAO.deleteById(dbTopicPO.getId());
|
||||
continue;
|
||||
}
|
||||
|
||||
topicDAO.updateById(TopicConverter.mergeAndOnlyMetadata2NewTopicPO(topic, dbTopicPO));
|
||||
}
|
||||
|
||||
// DB中没有的则插入DB
|
||||
for (Topic topic: presentTopicMap.values()) {
|
||||
try {
|
||||
topicDAO.insert(TopicConverter.mergeAndOnlyMetadata2NewTopicPO(topic, null));
|
||||
} catch (DuplicateKeyException dke) {
|
||||
// 忽略key冲突错误，多台KM可能同时做insert，所以可能出现key冲突
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int batchReplaceConfig(Long clusterPhyId, List<TopicConfig> topicConfigList) {
|
||||
int effectRow = 0;
|
||||
for (TopicConfig config: topicConfigList) {
|
||||
try {
|
||||
effectRow += topicDAO.updateConfig(ConvertUtil.obj2Obj(config, TopicPO.class));
|
||||
} catch (Exception e) {
|
||||
log.error("method=batchReplaceConfig||config={}||errMsg=exception!", config, e);
|
||||
}
|
||||
}
|
||||
|
||||
return effectRow;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result<Void> updatePartitionNum(Long clusterPhyId, String topicName, Integer partitionNum) {
|
||||
try {
|
||||
LambdaUpdateWrapper<TopicPO> lambdaUpdateWrapper = new LambdaUpdateWrapper<>();
|
||||
lambdaUpdateWrapper.eq(TopicPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaUpdateWrapper.eq(TopicPO::getTopicName, topicName);
|
||||
|
||||
TopicPO topicPO = new TopicPO();
|
||||
topicPO.setPartitionNum(partitionNum);
|
||||
|
||||
if (topicDAO.update(topicPO, lambdaUpdateWrapper) > 0){
|
||||
return Result.buildSuc();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("method=updatePartitionNum||clusterPhyId={}||topicName={}||partitionNum={}||errMsg=exception!", clusterPhyId, topicName, e);
|
||||
return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED);
|
||||
}
|
||||
|
||||
return Result.buildFrom(ResultStatus.MYSQL_OPERATE_FAILED);
|
||||
}
|
||||
|
||||
/**************************************************** private method ****************************************************/
|
||||
|
||||
private Result<List<Topic>> getTopicsFromAdminClient(ClusterPhy clusterPhy) {
|
||||
try {
|
||||
AdminClient adminClient = kafkaAdminClient.getClient(clusterPhy.getId());
|
||||
|
||||
ListTopicsResult listTopicsResult = adminClient.listTopics(new ListTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
|
||||
List<Topic> topicList = new ArrayList<>();
|
||||
|
||||
DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(listTopicsResult.names().get(), new DescribeTopicsOptions().timeoutMs(KafkaConstant.ADMIN_CLIENT_REQUEST_TIME_OUT_UNIT_MS));
|
||||
Map<String, TopicDescription> descriptionMap = describeTopicsResult.all().get();
|
||||
for (TopicDescription description: descriptionMap.values()) {
|
||||
topicList.add(TopicConverter.convert2Topic(clusterPhy.getId(), description));
|
||||
}
|
||||
|
||||
return Result.buildSuc(topicList);
|
||||
} catch (Exception e) {
|
||||
log.error("class=TopicServiceImpl||method=getTopicsFromAdminClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Result<List<Topic>> getTopicsFromZKClient(ClusterPhy clusterPhy) {
|
||||
try {
|
||||
|
||||
List<Topic> topicList = new ArrayList<>();
|
||||
|
||||
List<String> topicNameList = kafkaZKDAO.getChildren(clusterPhy.getId(), TopicsZNode.path(), false);
|
||||
for (String topicName: topicNameList) {
|
||||
topicList.add(kafkaZKDAO.getTopicMetadata(clusterPhy.getId(), topicName));
|
||||
}
|
||||
|
||||
return Result.buildSuc(topicList);
|
||||
} catch (Exception e) {
|
||||
log.error("class=TopicServiceImpl||method=getTopicsFromZKClient||clusterPhyId={}||errMsg=exception", clusterPhy.getId(), e);
|
||||
|
||||
return Result.buildFromRSAndMsg(ResultStatus.KAFKA_OPERATE_FAILED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Topic> getTopicsAndUpdateCache(Long clusterPhyId) {
|
||||
List<Topic> topicList = this.listTopicsFromDB(clusterPhyId);
|
||||
|
||||
Map<String, Topic> topicMap = topicList.stream().collect(Collectors.toMap(Topic::getTopicName, Function.identity()));
|
||||
topicsCache.put(clusterPhyId, topicMap);
|
||||
return topicMap;
|
||||
}
|
||||
|
||||
private TopicPO getTopicFromDB(Long clusterPhyId, String topicName) {
|
||||
LambdaQueryWrapper<TopicPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(TopicPO::getClusterPhyId, clusterPhyId);
|
||||
lambdaQueryWrapper.eq(TopicPO::getTopicName, topicName);
|
||||
|
||||
return topicDAO.selectOne(lambdaQueryWrapper);
|
||||
}
|
||||
|
||||
private List<TopicPO> getTopicsFromDB(Long clusterPhyId) {
|
||||
LambdaQueryWrapper<TopicPO> lambdaQueryWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaQueryWrapper.eq(TopicPO::getClusterPhyId, clusterPhyId);
|
||||
|
||||
return topicDAO.selectList(lambdaQueryWrapper);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version;
|
||||
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchQuery;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricLineVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.line.MetricMultiLinesVO;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public abstract class BaseMetricService extends BaseVersionControlService {
|
||||
private static final ILog LOGGER = LogFactory.getLog(BaseMetricService.class);
|
||||
|
||||
private List<String> metricNames = new ArrayList<>();
|
||||
private List<String> metricFields = new ArrayList<>();
|
||||
|
||||
@PostConstruct
|
||||
public void init(){
|
||||
initMetricFieldAndNameList();
|
||||
initRegisterVCHandler();
|
||||
}
|
||||
|
||||
protected void initMetricFieldAndNameList(){
|
||||
metricNames = listVersionControlItems().stream().map(v -> v.getName()).collect(Collectors.toList());
|
||||
metricFields = listMetricPOFields();
|
||||
}
|
||||
|
||||
protected abstract List<String> listMetricPOFields();
|
||||
|
||||
protected abstract void initRegisterVCHandler();
|
||||
|
||||
protected <T> List<MetricMultiLinesVO> metricMap2VO(Long clusterId,
|
||||
Map<String/*metric*/, Map<T, List<MetricPointVO>>> map){
|
||||
List<MetricMultiLinesVO> multiLinesVOS = new ArrayList<>();
|
||||
if (map == null || map.isEmpty()) {
|
||||
// 如果为空，则直接返回
|
||||
return multiLinesVOS;
|
||||
}
|
||||
|
||||
for(String metric : map.keySet()){
|
||||
try {
|
||||
MetricMultiLinesVO multiLinesVO = new MetricMultiLinesVO();
|
||||
multiLinesVO.setMetricName(metric);
|
||||
|
||||
List<MetricLineVO> metricLines = new ArrayList<>();
|
||||
|
||||
Map<T, List<MetricPointVO>> metricPointMap = map.get(metric);
|
||||
if(null == metricPointMap || metricPointMap.isEmpty()){continue;}
|
||||
for(Map.Entry<T, List<MetricPointVO>> entry : metricPointMap.entrySet()){
|
||||
MetricLineVO metricLineVO = new MetricLineVO();
|
||||
metricLineVO.setName(entry.getKey().toString());
|
||||
metricLineVO.setMetricName(metric);
|
||||
metricLineVO.setMetricPoints(entry.getValue());
|
||||
|
||||
metricLines.add(metricLineVO);
|
||||
}
|
||||
|
||||
multiLinesVO.setMetricLines(metricLines);
|
||||
multiLinesVOS.add(multiLinesVO);
|
||||
}catch (Exception e){
|
||||
LOGGER.error("method=metricMap2VO||cluster={}||msg=exception!", clusterId, e);
|
||||
}
|
||||
}
|
||||
|
||||
return multiLinesVOS;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查 str 是不是一个 metricName
|
||||
* @param str
|
||||
*/
|
||||
protected boolean isMetricName(String str){
|
||||
return metricNames.contains(str);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查 str 是不是一个 fieldName
|
||||
* @param str
|
||||
*/
|
||||
protected boolean isMetricField(String str){
|
||||
return metricFields.contains(str);
|
||||
}
|
||||
|
||||
protected void setQueryMetricFlag(SearchQuery query){
|
||||
if(null == query){return;}
|
||||
|
||||
String fieldName = query.getQueryName();
|
||||
|
||||
query.setMetric(isMetricName(fieldName));
|
||||
query.setField(isMetricField(fieldName));
|
||||
}
|
||||
|
||||
protected <T extends SearchQuery> void setQueryMetricFlag(List<T> matches){
|
||||
if(CollectionUtils.isEmpty(matches)){return;}
|
||||
|
||||
for (SearchQuery match : matches){
|
||||
setQueryMetricFlag(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.didiglobal.logi.log.ILog;
|
||||
import com.didiglobal.logi.log.LogFactory;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMethodInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.EnvUtil;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.DependsOn;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@DependsOn("versionControlService")
|
||||
public abstract class BaseVersionControlService {
|
||||
protected static final ILog LOGGER = LogFactory.getLog("METRIC_LOGGER");
|
||||
|
||||
@Autowired
|
||||
protected VersionControlService versionControlService;
|
||||
|
||||
/**
|
||||
* 多版本兼容类型
|
||||
* @return
|
||||
*/
|
||||
protected abstract VersionItemTypeEnum getVersionItemType();
|
||||
|
||||
/**
|
||||
* 注册版本兼容的执行方法，[minVersion,maxVersion) 是一个左闭右开的范围，即：minVersion<= kafkaClusterVersion < maxVersion
|
||||
* @param action
|
||||
* @param minVersion
|
||||
* @param maxVersion
|
||||
* @param methodName
|
||||
* @param func
|
||||
*/
|
||||
protected void registerVCHandler(String action,
|
||||
VersionEnum minVersion,
|
||||
VersionEnum maxVersion,
|
||||
String methodName,
|
||||
Function<VersionItemParam, Object> func) {
|
||||
versionControlService.registerHandler(getVersionItemType(), action, minVersion.getVersionL(), maxVersion.getVersionL(), methodName, func);
|
||||
}
|
||||
|
||||
protected void registerVCHandler(String methodName, Function<VersionItemParam, Object> func) {
|
||||
versionControlService.registerHandler(getVersionItemType(), methodName, func);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
protected Object doVCHandler(Long clusterPhyId, String action, VersionItemParam param) throws VCHandlerNotExistException {
|
||||
String methodName = getMethodName(clusterPhyId, action);
|
||||
Object ret = versionControlService.doHandler(getVersionItemType(), methodName, param);
|
||||
|
||||
if(!EnvUtil.isOnline()){
|
||||
LOGGER.info("method=doVCHandler||clusterId={}||action={}||methodName={}||type={}param={}||ret={}}!",
|
||||
clusterPhyId, action, methodName, getVersionItemType().getMessage(), JSON.toJSONString(param), JSON.toJSONString(ret));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
protected String getMethodName(Long clusterId, String action) {
|
||||
VersionControlItem item = versionControlService.getVersionControlItem(clusterId, getVersionItemType().getCode(), action);
|
||||
if (null == item) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (item.getExtend() instanceof VersionMethodInfo) {
|
||||
return ((VersionMethodInfo) item.getExtend()).getMethodName();
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
protected VersionJmxInfo getJMXInfo(Long clusterId, String action){
|
||||
VersionControlItem item = versionControlService.getVersionControlItem(clusterId, getVersionItemType().getCode(), action);
|
||||
if (null == item) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (item.getExtend() instanceof VersionJmxInfo) {
|
||||
return ((VersionJmxInfo) item.getExtend());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
protected List<VersionControlItem> listVersionControlItems(){
|
||||
List<VersionControlItem> controlItems = versionControlService.listVersionControlItem(getVersionItemType().getCode());
|
||||
if(CollectionUtils.isEmpty(controlItems)){
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
return controlItems;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public interface VersionControlMetricService {
|
||||
|
||||
/**
|
||||
* versionItemType
|
||||
* @return
|
||||
*/
|
||||
int versionItemType();
|
||||
|
||||
/**
|
||||
* init
|
||||
* @return
|
||||
*/
|
||||
<T extends VersionControlItem> List<T> init();
|
||||
|
||||
default <T extends VersionControlItem> Map<String, List<T>> initMap() {
|
||||
Map<String, List<T>> itemMap = new HashMap<>();
|
||||
|
||||
List<T> items = init();
|
||||
|
||||
for(T v : items){
|
||||
String metricName = v.getName();
|
||||
List<T> temp = itemMap.getOrDefault(metricName, new ArrayList<>());
|
||||
|
||||
temp.add(v);
|
||||
itemMap.put(metricName, temp);
|
||||
}
|
||||
|
||||
return itemMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public interface VersionControlService {
|
||||
|
||||
/**
|
||||
* registerHandler
|
||||
* @param typeEnum
|
||||
* @param method
|
||||
* @param func
|
||||
*/
|
||||
void registerHandler(VersionItemTypeEnum typeEnum, String method, Function<VersionItemParam, Object> func);
|
||||
|
||||
|
||||
/**
|
||||
* registerHandler
|
||||
* @param typeEnum
|
||||
* @param action
|
||||
* @param minVersion
|
||||
* @param maxVersion
|
||||
* @param methodName
|
||||
* @param func
|
||||
*/
|
||||
void registerHandler(VersionItemTypeEnum typeEnum, String action, long minVersion, long maxVersion,
|
||||
String methodName, Function<VersionItemParam, Object> func);
|
||||
/**
|
||||
* handler
|
||||
* @param typeEnum
|
||||
* @param methodName
|
||||
* @param param
|
||||
* @return
|
||||
*/
|
||||
Object doHandler(VersionItemTypeEnum typeEnum, String methodName, VersionItemParam param) throws VCHandlerNotExistException;
|
||||
|
||||
/**
|
||||
* 获取对应集群的版本兼容项
|
||||
* @param clusterId
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
List<VersionControlItem> listVersionControlItem(Long clusterId, Integer type);
|
||||
|
||||
/**
|
||||
* 获取对应type所有的的版本兼容项
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
List<VersionControlItem> listVersionControlItem(Integer type);
|
||||
|
||||
/**
|
||||
* 查询对应指标的版本兼容项
|
||||
* @param type
|
||||
* @param itemName
|
||||
* @return
|
||||
*/
|
||||
List<VersionControlItem> listVersionControlItem(Integer type, String itemName);
|
||||
|
||||
/**
|
||||
* 查询对应指标的版本兼容项
|
||||
* @param clusterId
|
||||
* @param type
|
||||
* @param itemName
|
||||
* @return
|
||||
*/
|
||||
VersionControlItem getVersionControlItem(Long clusterId, Integer type, String itemName);
|
||||
|
||||
/**
|
||||
* 判断 item 是否被 clusterId 对应的版本支持
|
||||
* @param clusterId
|
||||
* @param item
|
||||
* @return
|
||||
*/
|
||||
boolean isClusterSupport(Long clusterId, VersionControlItem item);
|
||||
|
||||
/**
|
||||
* 查询对应指标的版本兼容项
|
||||
* @param clusterId
|
||||
* @param type
|
||||
* @param itemNames
|
||||
* @return
|
||||
*/
|
||||
Map<String, VersionControlItem> getVersionControlItems(Long clusterId, Integer type, List<String> itemNames);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.fe;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.metrics.BaseMetricVersionMetric;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.WEB_OP;
|
||||
|
||||
/**
|
||||
* 多版本前端控件差异列表
|
||||
*/
|
||||
@Component
|
||||
public class FrontEndControlVersionItems extends BaseMetricVersionMetric {
|
||||
/**
|
||||
* 字段内容不可随意更改，因为前端会使用该字段进行页面控制
|
||||
*/
|
||||
|
||||
private static final String FE_TESTING_PRODUCER_HEADER = "FETestingProducerHeader";
|
||||
private static final String FE_TESTING_PRODUCER_COMPRESSION_TYPE_Z_STD = "FETestingProducerCompressionTypeZSTD";
|
||||
|
||||
private static final String FE_TESTING_CONSUMER_HEADER = "FETestingConsumerHeader";
|
||||
|
||||
private static final String FE_BROKERS_SPECIFIED_BROKER_CONFIG_LIST = "FEBrokersSpecifiedBrokerConfigList";
|
||||
private static final String FE_BROKERS_SPECIFIED_BROKER_DATA_LOGS_LIST = "FEBrokersSpecifiedBrokerDataLogsList";
|
||||
|
||||
private static final String FE_SECURITY_ACL_CREATE_PATTERN_TYPE_PREFIXED = "FESecurityAclCreatePatternTypePrefixed";
|
||||
private static final String FE_SECURITY_ACL_CREATE_RESOURCE_TYPE_TRANSACTIONAL_ID = "FESecurityAclCreateResourceTypeTransactionalId";
|
||||
private static final String FE_SECURITY_ACL_CREATE_RESOURCE_TYPE_DELEGATION_TOKEN = "FESecurityAclCreateResourceTypeDelegationToken";
|
||||
|
||||
public FrontEndControlVersionItems(){}
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return WEB_OP.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionControlItem> init(){
|
||||
List<VersionControlItem> itemList = new ArrayList<>();
|
||||
|
||||
// 测试-生产者-Header字段
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_0_11_0_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_TESTING_PRODUCER_HEADER).desc("测试-生产者-Header字段支持的版本"));
|
||||
|
||||
// 测试-生产者-ZStd字段
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_2_1_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_TESTING_PRODUCER_COMPRESSION_TYPE_Z_STD).desc("CompressionType中的zstd压缩格式"));
|
||||
|
||||
// 测试-消费者-Header字段
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_0_11_0_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_TESTING_CONSUMER_HEADER).desc("测试-消费者-Header字段支持的版本"));
|
||||
|
||||
// Brokers-具体Broker-Config列表
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_0_10_1_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_BROKERS_SPECIFIED_BROKER_CONFIG_LIST).desc("Brokers-具体Broker-Config列表"));
|
||||
|
||||
// Brokers-具体Broker-DataLogs列表
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_1_0_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_BROKERS_SPECIFIED_BROKER_DATA_LOGS_LIST).desc("Brokers-具体Broker-DataLogs列表"));
|
||||
|
||||
// Security-创建ACL-PatternType-Prefixed
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_2_0_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_SECURITY_ACL_CREATE_PATTERN_TYPE_PREFIXED).desc("Security-创建ACL-PatternType-Prefixed"));
|
||||
|
||||
// Security-创建ACL-ResourceType-TransactionalId
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_0_11_0_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_SECURITY_ACL_CREATE_RESOURCE_TYPE_TRANSACTIONAL_ID).desc("Security-创建ACL-ResourceType-TransactionalId"));
|
||||
|
||||
// Security-创建ACL-ResourceType-DelegationToken
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_1_1_0).maxVersion(VersionEnum.V_MAX)
|
||||
.name(FE_SECURITY_ACL_CREATE_RESOURCE_TYPE_DELEGATION_TOKEN).desc("Security-创建ACL-ResourceType-DelegationToken"));
|
||||
|
||||
return itemList;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.impl;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.param.VersionItemParam;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMethodInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.component.SpringTool;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum;
|
||||
import com.xiaojukeji.know.streaming.km.common.exception.VCHandlerNotExistException;
|
||||
import com.xiaojukeji.know.streaming.km.common.utils.VersionUtil;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.cluster.ClusterPhyService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlMetricService;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.DependsOn;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.function.Function;
|
||||
|
||||
@Slf4j
|
||||
@DependsOn("springTool")
|
||||
@Service("versionControlService")
|
||||
public class VersionControlServiceImpl implements VersionControlService {
|
||||
|
||||
@Autowired
|
||||
private ClusterPhyService clusterPhyService;
|
||||
|
||||
private final Map<Integer, List<VersionControlItem>> versionItemMap = new ConcurrentHashMap<>();
|
||||
private final Map<Integer, Map<String, List<VersionControlItem>>> versionItemMetricNameMap = new ConcurrentHashMap<>();
|
||||
|
||||
private final Map<String, Function<VersionItemParam, Object>> functionMap = new ConcurrentHashMap<>();
|
||||
|
||||
@PostConstruct
|
||||
public void init(){
|
||||
Map<String, VersionControlMetricService> itemMap = SpringTool.getBeansOfType( VersionControlMetricService.class);
|
||||
|
||||
for(VersionControlMetricService service : itemMap.values()){
|
||||
versionItemMap.put(service.versionItemType(), service.init());
|
||||
versionItemMetricNameMap.put(service.versionItemType(), service.initMap());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerHandler(VersionItemTypeEnum typeEnum, String methodName, Function<VersionItemParam, Object> func){
|
||||
functionMap.put(typeEnum.getCode() + "@" + methodName , func);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerHandler(VersionItemTypeEnum typeEnum, String action, long minVersion, long maxVersion,
|
||||
String methodName, Function<VersionItemParam, Object> func){
|
||||
Integer typeCode = typeEnum.getCode();
|
||||
|
||||
VersionMethodInfo versionMethodInfo = new VersionMethodInfo();
|
||||
versionMethodInfo.setMethodName(methodName);
|
||||
|
||||
VersionControlItem controlItem = new VersionControlItem();
|
||||
controlItem.name(action).minVersion(minVersion).maxVersion(maxVersion)
|
||||
.type(typeCode).extend(versionMethodInfo);
|
||||
|
||||
List<VersionControlItem> controlItems = (null == versionItemMap.get(typeCode)) ?
|
||||
new CopyOnWriteArrayList<>() : versionItemMap.get(typeCode);
|
||||
|
||||
controlItems.add(controlItem);
|
||||
versionItemMap.put(typeCode, controlItems);
|
||||
|
||||
Map<String, List<VersionControlItem>> itemMap = new HashMap<>();
|
||||
itemMap.put(action, controlItems);
|
||||
versionItemMetricNameMap.put(typeCode, itemMap);
|
||||
|
||||
functionMap.put(typeCode + "@" + methodName , func);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object doHandler(VersionItemTypeEnum typeEnum, String methodName, VersionItemParam param) throws VCHandlerNotExistException {
|
||||
Function<VersionItemParam, Object> func = functionMap.get(typeEnum.getCode() + "@" + methodName);
|
||||
if(null == func) {
|
||||
throw new VCHandlerNotExistException(typeEnum.getCode() + "@" + methodName);
|
||||
}
|
||||
|
||||
return func.apply(param);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionControlItem> listVersionControlItem(Long clusterId, Integer type) {
|
||||
String versionStr = clusterPhyService.getVersionFromCacheFirst(clusterId);
|
||||
long versionLong = VersionUtil.normailze(versionStr);
|
||||
|
||||
List<VersionControlItem> items = versionItemMap.get(type);
|
||||
if(CollectionUtils.isEmpty(items)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
List<VersionControlItem> versionControlItems = new ArrayList<>();
|
||||
for(VersionControlItem item : items){
|
||||
if(versionLong >= item.getMinVersion() && versionLong < item.getMaxVersion()){
|
||||
versionControlItems.add(item);
|
||||
}
|
||||
}
|
||||
|
||||
return versionControlItems;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionControlItem> listVersionControlItem(Integer type){
|
||||
return versionItemMap.get(type);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionControlItem> listVersionControlItem(Integer type, String itemName){
|
||||
Map<String, List<VersionControlItem>> listMap = versionItemMetricNameMap.get(type);
|
||||
return listMap.get(itemName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public VersionControlItem getVersionControlItem(Long clusterId, Integer type, String itemName) {
|
||||
List<VersionControlItem> items = listVersionControlItem(clusterId, type);
|
||||
|
||||
for(VersionControlItem item : items){
|
||||
if(itemName.equals(item.getName())){
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isClusterSupport(Long clusterId, VersionControlItem item){
|
||||
String versionStr = clusterPhyService.getVersionFromCacheFirst(clusterId);
|
||||
long versionLong = VersionUtil.normailze(versionStr);
|
||||
|
||||
return item.getMinVersion() <= versionLong && versionLong < item.getMaxVersion();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, VersionControlItem> getVersionControlItems(Long clusterId, Integer type, List<String> itemNames){
|
||||
Map<String, VersionControlItem> versionControlItemMap = new HashMap<>();
|
||||
|
||||
for(String itemName : itemNames){
|
||||
VersionControlItem item = getVersionControlItem(clusterId, type, itemName);
|
||||
if(null != item){
|
||||
versionControlItemMap.put(itemName, item);
|
||||
}
|
||||
}
|
||||
|
||||
return versionControlItemMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMethodInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionJmxInfo;
|
||||
import com.xiaojukeji.know.streaming.km.core.service.version.VersionControlMetricService;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.V_0_10_0_0;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum.V_MAX;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
public abstract class BaseMetricVersionMetric implements VersionControlMetricService {
|
||||
|
||||
public static final String BYTE_PER_SEC = "byte/s";
|
||||
|
||||
protected VersionMetricControlItem buildAllVersionsItem(){
|
||||
VersionMetricControlItem item = new VersionMetricControlItem();
|
||||
item.setType(versionItemType());
|
||||
item.setMinVersion(V_0_10_0_0.getVersionL());
|
||||
item.setMaxVersion(V_MAX.getVersionL());
|
||||
return item;
|
||||
}
|
||||
|
||||
protected VersionMetricControlItem buildAllVersionsItem(String metric, String unit){
|
||||
VersionMetricControlItem item = new VersionMetricControlItem();
|
||||
item.setType(versionItemType());
|
||||
item.setName(metric);
|
||||
item.setUnit(unit);
|
||||
item.setMinVersion(V_0_10_0_0.getVersionL());
|
||||
item.setMaxVersion(V_MAX.getVersionL());
|
||||
return item;
|
||||
}
|
||||
|
||||
protected VersionMetricControlItem buildItem(String metric, String unit){
|
||||
VersionMetricControlItem item = new VersionMetricControlItem();
|
||||
item.setType(versionItemType());
|
||||
item.setName(metric);
|
||||
item.setUnit(unit);
|
||||
return item;
|
||||
}
|
||||
|
||||
protected VersionMetricControlItem buildItem(){
|
||||
VersionMetricControlItem item = new VersionMetricControlItem();
|
||||
item.setType(versionItemType());
|
||||
return item;
|
||||
}
|
||||
|
||||
protected VersionMethodInfo buildMethodExtend(String methodName){
|
||||
VersionMethodInfo versionMethodInfo = new VersionMethodInfo();
|
||||
versionMethodInfo.setMethodName(methodName);
|
||||
return versionMethodInfo;
|
||||
}
|
||||
|
||||
protected VersionJmxInfo buildJMXMethodExtend(String methodName){
|
||||
VersionJmxInfo jmxExtendInfo = new VersionJmxInfo();
|
||||
jmxExtendInfo.setMethodName(methodName);
|
||||
return jmxExtendInfo;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_BROKER;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxName.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.broker.impl.BrokerMetricServiceImpl.*;
|
||||
|
||||
@Component
|
||||
public class BrokerMetricVersionItems extends BaseMetricVersionMetric {
|
||||
|
||||
public static final String BROKER_METRIC_HEALTH_SCORE = "HealthScore";
|
||||
public static final String BROKER_METRIC_HEALTH_CHECK_PASSED = "HealthCheckPassed";
|
||||
public static final String BROKER_METRIC_HEALTH_CHECK_TOTAL = "HealthCheckTotal";
|
||||
public static final String BROKER_METRIC_TOTAL_REQ_QUEUE = "TotalRequestQueueSize";
|
||||
public static final String BROKER_METRIC_TOTAL_RES_QUEUE = "TotalResponseQueueSize";
|
||||
public static final String BROKER_METRIC_REP_BYTES_IN = "ReplicationBytesIn";
|
||||
public static final String BROKER_METRIC_REP_BYTES_OUT = "ReplicationBytesOut";
|
||||
public static final String BROKER_METRIC_MESSAGE_IN = "MessagesIn";
|
||||
public static final String BROKER_METRIC_TOTAL_PRODUCE_REQ = "TotalProduceRequests";
|
||||
public static final String BROKER_METRIC_NETWORK_RPO_AVG_IDLE = "NetworkProcessorAvgIdle";
|
||||
public static final String BROKER_METRIC_REQ_AVG_IDLE = "RequestHandlerAvgIdle";
|
||||
public static final String BROKER_METRIC_UNDER_REPLICATE_PARTITION = "PartitionURP";
|
||||
public static final String BROKER_METRIC_CONNECTION_COUNT = "ConnectionsCount";
|
||||
public static final String BROKER_METRIC_BYTES_IN = "BytesIn";
|
||||
public static final String BROKER_METRIC_BYTES_IN_5_MIN = "BytesIn_min_5";
|
||||
public static final String BROKER_METRIC_BYTES_IN_15_MIN = "BytesIn_min_15";
|
||||
public static final String BROKER_METRIC_REASSIGNMENT_BYTES_IN = "ReassignmentBytesIn";
|
||||
public static final String BROKER_METRIC_BYTES_OUT = "BytesOut";
|
||||
public static final String BROKER_METRIC_BYTES_OUT_5_MIN = "BytesOut_min_5";
|
||||
public static final String BROKER_METRIC_BYTES_OUT_15_MIN = "BytesOut_min_15";
|
||||
public static final String BROKER_METRIC_REASSIGNMENT_BYTES_OUT = "ReassignmentBytesOut";
|
||||
public static final String BROKER_METRIC_PARTITIONS = "Partitions";
|
||||
public static final String BROKER_METRIC_PARTITIONS_SKEW = "PartitionsSkew";
|
||||
public static final String BROKER_METRIC_LEADERS = "Leaders";
|
||||
public static final String BROKER_METRIC_LEADERS_SKEW = "LeadersSkew";
|
||||
public static final String BROKER_METRIC_LOG_SIZE = "LogSize";
|
||||
public static final String BROKER_METRIC_ACTIVE_CONTROLLER_COUNT = "ActiveControllerCount";
|
||||
public static final String BROKER_METRIC_ALIVE = "Alive";
|
||||
public static final String BROKER_METRIC_COLLECT_COST_TIME = Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME;
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return METRIC_BROKER.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionMetricControlItem> init(){
|
||||
List<VersionMetricControlItem> items = new ArrayList<>();
|
||||
|
||||
// HealthScore 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_HEALTH_SCORE).unit("分").desc("健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod(BROKER_METHOD_GET_HEALTH_SCORE));
|
||||
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_HEALTH_CHECK_PASSED ).unit("个").desc("健康检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod(BROKER_METHOD_GET_HEALTH_SCORE));
|
||||
|
||||
items.add(buildAllVersionsItem()
|
||||
.name( BROKER_METRIC_HEALTH_CHECK_TOTAL ).unit("个").desc("健康检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( BROKER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
// TotalRequestQueueSize 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_TOTAL_REQ_QUEUE).unit("个").desc("Broker的请求队列大小").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_NETWORK_REQUEST_QUEUE ).jmxAttribute(VALUE)));
|
||||
|
||||
// TotalResponseQueueSize 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_TOTAL_RES_QUEUE).unit("个").desc("Broker的应答队列大小").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_NETWORK_RESPONSE_QUEUE ).jmxAttribute(VALUE)));
|
||||
|
||||
// MessagesIn 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_MESSAGE_IN).unit("条/s").desc("Broker的每秒消息流入条数").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_MESSAGES_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// TotalProduceRequests 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_TOTAL_PRODUCE_REQ).unit("个/s").desc("Broker上Produce的每秒请求数").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_PRODUCES_REQUEST ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// NetworkProcessorAvgIdle 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_NETWORK_RPO_AVG_IDLE).unit("%").desc("Broker的网络处理器的空闲百分比").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_NETWORK_PROCESSOR_AVG_IDLE ).jmxAttribute(VALUE)));
|
||||
|
||||
// RequestHandlerAvgIdle 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_REQ_AVG_IDLE).unit("%").desc("Broker上请求处理器的空闲百分比").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_REQUEST_AVG_IDLE ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// PartitionURP 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_UNDER_REPLICATE_PARTITION).unit("个").desc("Broker上的未同步的副本的个数").category(CATEGORY_PARTITION)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_UNDER_REP_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// connectionsCount 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_CONNECTION_COUNT).unit("个").desc("Broker上网络链接的个数").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_CONNECTION_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_SOCKET_CONNECTIONS ).jmxAttribute(CONNECTION_COUNT)));
|
||||
|
||||
// BytesInPerSec 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_BYTES_IN).unit(BYTE_PER_SEC).desc("Broker的每秒数据写入量").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
items.add(buildAllVersionsItem(BROKER_METRIC_BYTES_IN_5_MIN, BYTE_PER_SEC)
|
||||
.desc("Broker的每秒数据写入量，5分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_5)));
|
||||
|
||||
items.add(buildAllVersionsItem(BROKER_METRIC_BYTES_IN_15_MIN, BYTE_PER_SEC)
|
||||
.desc("Broker的每秒数据写入量，15分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_15)));
|
||||
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_REASSIGNMENT_BYTES_IN).unit(BYTE_PER_SEC)
|
||||
.desc("Broker的每秒数据迁移写入量").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_REASSIGNMENT_BYTE_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// BytesInPerSec 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_BYTES_OUT).unit(BYTE_PER_SEC).desc("Broker的每秒数据流出量").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
items.add(buildAllVersionsItem(BROKER_METRIC_BYTES_OUT_5_MIN, BYTE_PER_SEC)
|
||||
.desc("Broker的每秒数据流出量，5分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_5)));
|
||||
|
||||
items.add(buildAllVersionsItem(BROKER_METRIC_BYTES_OUT_15_MIN, BYTE_PER_SEC)
|
||||
.desc("Broker的每秒数据流出量，15分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_15)));
|
||||
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_REASSIGNMENT_BYTES_OUT).unit(BYTE_PER_SEC)
|
||||
.desc("Broker的每秒数据迁移流出量").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_REASSIGNMENT_BYTE_OUT ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// Partitions 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_PARTITIONS).unit("个").desc("Broker上的Partition个数").category(CATEGORY_PARTITION)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// PartitionsSkew 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_PARTITIONS_SKEW).unit("%").desc("Broker上的Partitions倾斜度").category(CATEGORY_PARTITION)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_PARTITIONS_SKEW )
|
||||
.jmxObjectName( JMX_SERVER_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// Leaders 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_LEADERS).unit("个").desc("Broker上的Leaders个数").category(CATEGORY_PARTITION)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_LEADERS ).jmxAttribute(VALUE)));
|
||||
|
||||
// LeadersSkew 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_LEADERS_SKEW).unit("%").desc("Broker上的Leaders倾斜度").category(CATEGORY_PARTITION)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_LEADERS_SKEW )
|
||||
.jmxObjectName( JMX_SERVER_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// LogSize 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_LOG_SIZE).unit("byte").desc("Broker上的消息容量大小").category(CATEGORY_PARTITION)
|
||||
.extendMethod(BROKER_METHOD_GET_LOG_SIZE));
|
||||
|
||||
// ActiveControllerCount 指标
|
||||
items.add(buildAllVersionsItem(BROKER_METRIC_ACTIVE_CONTROLLER_COUNT, "个").desc("Broker是否为controller").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_CONTROLLER_ACTIVE_COUNT ).jmxAttribute(VALUE)));
|
||||
|
||||
// ActiveControllerCount 指标
|
||||
items.add(buildAllVersionsItem(BROKER_METRIC_ALIVE, "是/否").desc("Broker是否存活，1：存活；0：没有存活").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildMethodExtend( BROKER_METHOD_IS_BROKER_ALIVE )));
|
||||
|
||||
// BROKER_METRIC_REP_BYTES_IN 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_REP_BYTES_IN).unit(BYTE_PER_SEC).desc("Broker 的 ReplicationBytesIn").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_REPLICATION_BYTES_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// BROKER_METRIC_REP_BYTES_OUT 指标
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_REP_BYTES_OUT).unit(BYTE_PER_SEC).desc("Broker 的 ReplicationBytesOut").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( BROKER_METHOD_GET_METRIC_FROM_KAFKA_BY_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_REPLICATION_BYTES_OUT ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
items.add(buildAllVersionsItem()
|
||||
.name(BROKER_METRIC_COLLECT_COST_TIME).unit("秒").desc("采集Broker指标的耗时").category(CATEGORY_PERFORMANCE)
|
||||
.extendMethod(BROKER_METHOD_DO_NOTHING));
|
||||
return items;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import com.xiaojukeji.know.streaming.km.common.enums.version.VersionEnum;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_CLUSTER;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxName.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.cluster.impl.ClusterMetricServiceImpl.*;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Component
|
||||
public class ClusterMetricVersionItems extends BaseMetricVersionMetric {
|
||||
|
||||
public static final String CLUSTER_METRIC_HEALTH_SCORE = "HealthScore";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_PASSED = "HealthCheckPassed";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_TOTAL = "HealthCheckTotal";
|
||||
public static final String CLUSTER_METRIC_HEALTH_SCORE_TOPICS = "HealthScore_Topics";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS = "HealthCheckPassed_Topics";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS = "HealthCheckTotal_Topics";
|
||||
public static final String CLUSTER_METRIC_HEALTH_SCORE_BROKERS = "HealthScore_Brokers";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS = "HealthCheckPassed_Brokers";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS = "HealthCheckTotal_Brokers";
|
||||
public static final String CLUSTER_METRIC_HEALTH_SCORE_GROUPS = "HealthScore_Groups";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS = "HealthCheckPassed_Groups";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS = "HealthCheckTotal_Groups";
|
||||
public static final String CLUSTER_METRIC_HEALTH_SCORE_CLUSTER = "HealthScore_Cluster";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_PASSED_CLUSTER = "HealthCheckPassed_Cluster";
|
||||
public static final String CLUSTER_METRIC_HEALTH_CHECK_TOTAL_CLUSTER = "HealthCheckTotal_Cluster";
|
||||
public static final String CLUSTER_METRIC_TOTAL_REQ_QUEUE_SIZE = "TotalRequestQueueSize";
|
||||
public static final String CLUSTER_METRIC_TOTAL_RES_QUEUE_SIZE = "TotalResponseQueueSize";
|
||||
public static final String CLUSTER_METRIC_EVENT_QUEUE_SIZE = "EventQueueSize";
|
||||
public static final String CLUSTER_METRIC_ACTIVE_CONTROLLER_COUNT = "ActiveControllerCount";
|
||||
public static final String CLUSTER_METRIC_TOTAL_PRODUCE_REQ = "TotalProduceRequests";
|
||||
public static final String CLUSTER_METRIC_TOTAL_LOG_SIZE = "TotalLogSize";
|
||||
public static final String CLUSTER_METRIC_CONNECTIONS = "ConnectionsCount";
|
||||
public static final String CLUSTER_METRIC_ZOOKEEPERS = "Zookeepers";
|
||||
public static final String CLUSTER_METRIC_ZOOKEEPERS_AVAILABLE = "ZookeepersAvailable";
|
||||
public static final String CLUSTER_METRIC_BROKERS = "Brokers";
|
||||
public static final String CLUSTER_METRIC_BROKERS_ALIVE = "BrokersAlive";
|
||||
public static final String CLUSTER_METRIC_BROKERS_NOT_ALIVE = "BrokersNotAlive";
|
||||
public static final String CLUSTER_METRIC_REPLICAS = "Replicas";
|
||||
public static final String CLUSTER_METRIC_TOPICS = "Topics";
|
||||
public static final String CLUSTER_METRIC_PARTITIONS = "Partitions";
|
||||
public static final String CLUSTER_METRIC_PARTITIONS_NO_LEADER = "PartitionNoLeader";
|
||||
public static final String CLUSTER_METRIC_PARTITION_MIN_ISR_S = "PartitionMinISR_S";
|
||||
public static final String CLUSTER_METRIC_PARTITION_MIN_ISR_E = "PartitionMinISR_E";
|
||||
public static final String CLUSTER_METRIC_PARTITION_URP = "PartitionURP";
|
||||
public static final String CLUSTER_METRIC_MESSAGES_IN = "MessagesIn";
|
||||
public static final String CLUSTER_METRIC_MESSAGES = "Messages";
|
||||
public static final String CLUSTER_METRIC_LEADER_MESSAGES = "LeaderMessages";
|
||||
public static final String CLUSTER_METRIC_BYTES_IN = "BytesIn";
|
||||
public static final String CLUSTER_METRIC_BYTES_IN_5_MIN = "BytesIn_min_5";
|
||||
public static final String CLUSTER_METRIC_BYTES_IN_15_MIN = "BytesIn_min_15";
|
||||
public static final String CLUSTER_METRIC_BYTES_OUT = "BytesOut";
|
||||
public static final String CLUSTER_METRIC_BYTES_OUT_5_MIN = "BytesOut_min_5";
|
||||
public static final String CLUSTER_METRIC_BYTES_OUT_15_MIN = "BytesOut_min_15";
|
||||
public static final String CLUSTER_METRIC_GROUP = "Groups";
|
||||
public static final String CLUSTER_METRIC_GROUP_ACTIVES = "GroupActives";
|
||||
public static final String CLUSTER_METRIC_GROUP_EMPTYS = "GroupEmptys";
|
||||
public static final String CLUSTER_METRIC_GROUP_REBALANCES = "GroupRebalances";
|
||||
public static final String CLUSTER_METRIC_GROUP_DEADS = "GroupDeads";
|
||||
public static final String CLUSTER_METRIC_ALIVE = "Alive";
|
||||
|
||||
public static final String CLUSTER_METRIC_ACL_ENABLE = "AclEnable";
|
||||
public static final String CLUSTER_METRIC_ACLS = "Acls";
|
||||
public static final String CLUSTER_METRIC_ACL_USERS = "AclUsers";
|
||||
public static final String CLUSTER_METRIC_ACL_TOPICS = "AclTopics";
|
||||
public static final String CLUSTER_METRIC_ACL_GROUPS = "AclGroups";
|
||||
|
||||
public static final String CLUSTER_METRIC_JOB = "Jobs";
|
||||
public static final String CLUSTER_METRIC_JOB_RUNNING = "JobsRunning";
|
||||
public static final String CLUSTER_METRIC_JOB_WAITING = "JobsWaiting";
|
||||
public static final String CLUSTER_METRIC_JOB_SUCCESS = "JobsSuccess";
|
||||
public static final String CLUSTER_METRIC_JOB_FAILED = "JobsFailed";
|
||||
|
||||
public static final String CLUSTER_METRIC_COLLECT_COST_TIME = Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME;
|
||||
|
||||
public ClusterMetricVersionItems(){}
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return METRIC_CLUSTER.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionMetricControlItem> init(){
|
||||
List<VersionMetricControlItem> itemList = new ArrayList<>();
|
||||
|
||||
// HealthScore 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_SCORE).unit("分").desc("集群总体的健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod(CLUSTER_METHOD_GET_HEALTH_SCORE));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_PASSED).unit("个").desc("集群总体健康检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_TOTAL).unit("个").desc("集群总体健康检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_SCORE_TOPICS).unit("分").desc("集群Topics的健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_PASSED_TOPICS).unit("个").desc("集群Topics健康检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_TOPICS).unit("个").desc("集群Topics健康检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_SCORE_BROKERS).unit("分").desc("集群Brokers的健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_PASSED_BROKERS).unit("个").desc("集群Brokers健康检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_BROKERS).unit("个").desc("集群Brokers健康检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_SCORE_GROUPS).unit("分").desc("集群Groups的健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_PASSED_GROUPS).unit("个").desc("集群Groups健康检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_GROUPS).unit("个").desc("集群Groups健康检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_SCORE_CLUSTER).unit("分").desc("集群自身的健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_PASSED_CLUSTER).unit("个").desc("集群自身健康检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_HEALTH_CHECK_TOTAL_CLUSTER).unit("个").desc("集群自身健康检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( CLUSTER_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
// TotalRequestQueueSize 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_TOTAL_REQ_QUEUE_SIZE).unit("个").desc("集群的中总的请求队列数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_NETWORK_REQUEST_QUEUE ).jmxAttribute(VALUE)));
|
||||
|
||||
// TotalResponseQueueSize 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_TOTAL_RES_QUEUE_SIZE).unit("个").desc("集群的 TotalResponseQueueSize").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_NETWORK_RESPONSE_QUEUE ).jmxAttribute(VALUE)));
|
||||
|
||||
// EventQueueSize 指标
|
||||
itemList.add(buildItem().minVersion(VersionEnum.V_2_0_0).maxVersion(VersionEnum.V_MAX).category(CATEGORY_CLUSTER)
|
||||
.name(CLUSTER_METRIC_EVENT_QUEUE_SIZE).unit("个").desc("集群中controller的EventQueueSize大小")
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_CONTROLLER_JMX )
|
||||
.jmxObjectName( JMX_CONTROLLER_EVENT_QUEUE_SIZE ).jmxAttribute(VALUE)));
|
||||
|
||||
// ActiveControllerCount 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ACTIVE_CONTROLLER_COUNT).unit("个").desc("集群中ActiveControllerCount大小").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_CONTROLLER_ACTIVE_COUNT ).jmxAttribute(VALUE)));
|
||||
|
||||
// TotalProduceRequests 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_TOTAL_PRODUCE_REQ).unit("个/s").desc("集群中的Produce每秒请求数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_PRODUCES_REQUEST ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// TotalLogSize 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_TOTAL_LOG_SIZE).unit("byte").desc("集群总的已使用的磁盘大小").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_TOTAL_LOG_SIZE )));
|
||||
|
||||
// Connections 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_CONNECTIONS).unit("个").desc("集群的连接(Connections)个数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName(JMX_SERVER_SOCKET_CONNECTIONS).jmxAttribute(VALUE)));
|
||||
|
||||
// ZKs 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ZOOKEEPERS).unit("个").desc("集群中存活的zk节点个数").category(CATEGORY_CLUSTER)
|
||||
.extend(buildMethodExtend( CLUSTER_METHOD_GET_ZK_COUNT )));
|
||||
|
||||
// ZK 是否合法，即可连接
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ZOOKEEPERS_AVAILABLE).unit("是/否").desc("ZK地址是否合法").category(CATEGORY_CLUSTER)
|
||||
.extend(buildMethodExtend(CLUSTER_METHOD_GET_ZK_AVAILABLE)));
|
||||
|
||||
// Brokers 总数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BROKERS).unit("个").desc("集群的broker的总数").category(CATEGORY_BROKER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_BROKERS_COUNT )));
|
||||
|
||||
// Brokers 存活数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BROKERS_ALIVE).unit("个").desc("集群的broker的存活数").category(CATEGORY_BROKER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_BROKERS_ALIVE_COUNT )));
|
||||
|
||||
// Brokers 未存活数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BROKERS_NOT_ALIVE).unit("个").desc("集群的broker的未存活数").category(CATEGORY_BROKER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_BROKERS_NOT_ALIVE_COUNT )));
|
||||
|
||||
// Replicas 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_REPLICAS).unit("个").desc("集群中Replica的总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_REPLICAS_COUNT )));
|
||||
|
||||
// Topics 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_TOPICS).unit("个").desc("集群中Topic的总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_TOPIC_SIZE )));
|
||||
|
||||
// Partitions 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_PARTITIONS).unit("个").desc("集群的Partitions总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_PARTITION_SIZE )));
|
||||
|
||||
// PartitionNoLeader 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_PARTITIONS_NO_LEADER).unit("个").desc("集群中的PartitionNoLeader总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_PARTITION_NO_LEADER_SIZE )));
|
||||
|
||||
// 小于 PartitionMinISR 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_PARTITION_MIN_ISR_S).unit("个").desc("集群中的小于PartitionMinISR总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_UNDER_MIN_ISR_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// 等于 PartitionMinISR 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_PARTITION_MIN_ISR_E).unit("个").desc("集群中的PartitionMinISR总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_AT_MIN_ISR_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// PartitionURP 个数指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_PARTITION_URP).unit("个").desc("集群中的未同步的总数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_UNDER_REP_PARTITIONS ).jmxAttribute(VALUE)));
|
||||
|
||||
// MessagesIn 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_MESSAGES_IN).unit("条/s").desc("集群每条消息写入条数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_MESSAGES_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// LeaderMessages 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_LEADER_MESSAGES).unit("条").desc("集群中leader总的消息条数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_MESSAGE_SIZE )));
|
||||
|
||||
// Messages 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_MESSAGES).unit("条").desc("集群总的消息条数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_MESSAGE_SIZE )));
|
||||
|
||||
// BytesInPerSec 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BYTES_IN).unit(BYTE_PER_SEC).desc("集群的每秒写入字节数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BYTES_IN_5_MIN).unit(BYTE_PER_SEC).desc("集群的每秒写入字节数，5分钟均值").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_5)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BYTES_IN_15_MIN).unit(BYTE_PER_SEC).desc("集群的每秒写入字节数，15分钟均值").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_15)));
|
||||
|
||||
// BytesOutPerSec 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BYTES_OUT).unit(BYTE_PER_SEC).desc("集群的每秒流出字节数").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BYTES_OUT_5_MIN).unit(BYTE_PER_SEC).desc("集群的每秒流出字节数，5分钟均值").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_5)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_BYTES_OUT_15_MIN).unit(BYTE_PER_SEC).desc("集群的每秒流出字节数，15分钟均值").category(CATEGORY_CLUSTER)
|
||||
.extend( buildJMXMethodExtend( CLUSTER_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKERS_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_15)));
|
||||
|
||||
// 集群维度-Group相关指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_GROUP).unit("个").desc("集群中Group的总数").category(CATEGORY_CONSUMER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_GROUP_COUNT )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_GROUP_ACTIVES).unit("个").desc("集群中ActiveGroup的总数").category(CATEGORY_CONSUMER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_GROUP_ACTIVE_COUNT )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_GROUP_EMPTYS).unit("个").desc("集群中EmptyGroup的总数").category(CATEGORY_CONSUMER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_GROUP_EMPTY_COUNT )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_GROUP_REBALANCES).unit("个").desc("集群中reBalanceGroup的总数").category(CATEGORY_CONSUMER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_GROUP_REBALANCED_COUNT )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_GROUP_DEADS).unit("个").desc("集群中DeadGroup的总数").category(CATEGORY_CONSUMER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_GROUP_DEAD_COUNT )));
|
||||
|
||||
// 集群维度-alive
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ALIVE).unit("是/否").desc("集群是否存活，1：存活；0：没有存活").category(CATEGORY_CLUSTER)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_ALIVE )));
|
||||
|
||||
// 集群维度-ACL相关指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ACL_ENABLE).unit("是/否").desc("集群是否开启Acl，1：是；0：否").category(CATEGORY_SECURITY)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_ACL_ENABLE )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ACLS).unit("个").desc("ACL数").category(CATEGORY_SECURITY)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_ACLS )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ACL_USERS).unit("个").desc("ACL-KafkaUser数").category(CATEGORY_SECURITY)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_ACL_USERS )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ACL_TOPICS).unit("个").desc("ACL-Topic数").category(CATEGORY_SECURITY)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_ACL_TOPICS )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_ACL_GROUPS).unit("个").desc("ACL-Group数").category(CATEGORY_SECURITY)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_ACL_GROUPS )));
|
||||
|
||||
// 集群维度-Jobs相关指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_JOB).unit("个").desc("集群任务总数").category(CATEGORY_JOB)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_JOBS )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_JOB_RUNNING).unit("个").desc("集群running任务总数").category(CATEGORY_JOB)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_JOBS_RUNNING )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_JOB_WAITING).unit("个").desc("集群waiting任务总数").category(CATEGORY_JOB)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_JOBS_WAITING )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_JOB_SUCCESS).unit("个").desc("集群success任务总数").category(CATEGORY_JOB)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_JOBS_SUCCESS )));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_JOB_FAILED).unit("个").desc("集群failed任务总数").category(CATEGORY_JOB)
|
||||
.extend( buildMethodExtend( CLUSTER_METHOD_GET_JOBS_FAILED )));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(CLUSTER_METRIC_COLLECT_COST_TIME).unit("秒").desc("采集Cluster指标的耗时").category(CATEGORY_PERFORMANCE)
|
||||
.extendMethod(CLUSTER_METHOD_DO_NOTHING));
|
||||
return itemList;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMethodInfo;
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_GROUP;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.group.impl.GroupMetricServiceImpl.*;
|
||||
|
||||
@Component
|
||||
public class GroupMetricVersionItems extends BaseMetricVersionMetric {
|
||||
|
||||
public static final String GROUP_METRIC_HEALTH_SCORE = "HealthScore";
|
||||
public static final String GROUP_METRIC_HEALTH_CHECK_PASSED = "HealthCheckPassed";
|
||||
public static final String GROUP_METRIC_HEALTH_CHECK_TOTAL = "HealthCheckTotal";
|
||||
public static final String GROUP_METRIC_OFFSET_CONSUMED = "OffsetConsumed";
|
||||
public static final String GROUP_METRIC_LOG_END_OFFSET = "LogEndOffset";
|
||||
public static final String GROUP_METRIC_LAG = "Lag";
|
||||
public static final String GROUP_METRIC_STATE = "State";
|
||||
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return METRIC_GROUP.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionMetricControlItem> init(){
|
||||
List<VersionMetricControlItem> itemList = new ArrayList<>();
|
||||
|
||||
// HealthScore 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_HEALTH_SCORE).unit("分").desc("健康分")
|
||||
.extendMethod( GROUP_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_HEALTH_CHECK_PASSED ).unit("个").desc("健康检查通过数")
|
||||
.extendMethod( GROUP_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_HEALTH_CHECK_TOTAL ).unit("个").desc("健康检查总数")
|
||||
.extendMethod( GROUP_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
// OffsetConsumed 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_OFFSET_CONSUMED).unit("条").desc("Consumer的offset")
|
||||
.extend(new VersionMethodInfo()
|
||||
.methodName( GROUP_METHOD_GET_LAG_RELEVANT_FROM_ADMIN_CLIENT )));
|
||||
|
||||
// LogEndOffset 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_LOG_END_OFFSET).unit("条").desc("Group的LogEndOffset")
|
||||
.extendMethod(GROUP_METHOD_GET_LAG_RELEVANT_FROM_ADMIN_CLIENT));
|
||||
|
||||
// Lag 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_LAG).unit("条").desc("Group消费者的Lag数")
|
||||
.extendMethod( GROUP_METHOD_GET_LAG_RELEVANT_FROM_ADMIN_CLIENT));
|
||||
|
||||
// State 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(GROUP_METRIC_STATE ).unit("个").desc("Group组的状态")
|
||||
.extendMethod( GROUP_METHOD_GET_STATE ));
|
||||
|
||||
return itemList;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_PARTITION;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.RATE_MIN_1;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.VALUE;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxName.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.partition.impl.PartitionMetricServiceImpl.*;
|
||||
|
||||
/**
|
||||
* @author didi
|
||||
*/
|
||||
@Component
|
||||
public class PartitionMetricVersionItems extends BaseMetricVersionMetric {
|
||||
|
||||
public static final String PARTITION_METRIC_LOG_END_OFFSET = "LogEndOffset";
|
||||
public static final String PARTITION_METRIC_LOG_START_OFFSET = "LogStartOffset";
|
||||
public static final String PARTITION_METRIC_MESSAGES = "Messages";
|
||||
|
||||
public static final String PARTITION_METRIC_BYTES_IN = "BytesIn";
|
||||
public static final String PARTITION_METRIC_BYTES_OUT = "BytesOut";
|
||||
public static final String PARTITION_METRIC_LOG_SIZE = "LogSize";
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return METRIC_PARTITION.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionMetricControlItem> init(){
|
||||
List<VersionMetricControlItem> itemList = new ArrayList<>();
|
||||
|
||||
// LogEndOffset 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(PARTITION_METRIC_LOG_END_OFFSET).unit("条").desc("Partition中Leader副本的LogEndOffset")
|
||||
.extendMethod(PARTITION_METHOD_GET_OFFSET_RELEVANT_METRICS));
|
||||
|
||||
// LogStartOffset 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(PARTITION_METRIC_LOG_START_OFFSET).unit("条").desc("Partition中Leader副本的LogStartOffset")
|
||||
.extendMethod(PARTITION_METHOD_GET_OFFSET_RELEVANT_METRICS));
|
||||
|
||||
// Messages
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(PARTITION_METRIC_MESSAGES).unit("条").desc("Partition中Leader副本的消息条数")
|
||||
.extendMethod(PARTITION_METHOD_GET_OFFSET_RELEVANT_METRICS));
|
||||
|
||||
// BytesIn
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(PARTITION_METRIC_BYTES_IN).unit(BYTE_PER_SEC).desc("Partition的BytesIn")
|
||||
.extend( buildJMXMethodExtend(PARTITION_METHOD_GET_TOPIC_AVG_METRIC_MESSAGES)
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_1) ));
|
||||
|
||||
// BytesOut
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(PARTITION_METRIC_BYTES_OUT).unit(BYTE_PER_SEC).desc("Partition的BytesOut")
|
||||
.extend( buildJMXMethodExtend(PARTITION_METHOD_GET_TOPIC_AVG_METRIC_MESSAGES)
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_1) ));
|
||||
|
||||
// LogSize
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(PARTITION_METRIC_LOG_SIZE).unit("byte").desc("Partition的LogSize")
|
||||
.extend( buildJMXMethodExtend(PARTITION_METHOD_GET_METRIC_FROM_JMX)
|
||||
.jmxObjectName( JMX_LOG_LOG_SIZE ).jmxAttribute(VALUE) ));
|
||||
return itemList;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem.CATEGORY_PERFORMANCE;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_REPLICATION;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxName.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.replica.impl.ReplicaMetricServiceImpl.*;
|
||||
|
||||
@Component
|
||||
public class ReplicaMetricVersionItems extends BaseMetricVersionMetric {
|
||||
|
||||
public static final String REPLICATION_METRIC_LOG_END_OFFSET = "LogEndOffset";
|
||||
public static final String REPLICATION_METRIC_LOG_START_OFFSET = "LogStartOffset";
|
||||
public static final String REPLICATION_METRIC_MESSAGES = "Messages";
|
||||
public static final String REPLICATION_METRIC_LOG_SIZE = "LogSize";
|
||||
public static final String REPLICATION_METRIC_IN_SYNC = "InSync";
|
||||
public static final String REPLICATION_METRIC_COLLECT_COST_TIME = Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME;
|
||||
|
||||
public ReplicaMetricVersionItems(){}
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return METRIC_REPLICATION.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionMetricControlItem> init(){
|
||||
List<VersionMetricControlItem> itemList = new ArrayList<>();
|
||||
|
||||
// LogEndOffset 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(REPLICATION_METRIC_LOG_END_OFFSET).unit("条").desc("副本的LogEndOffset")
|
||||
.extend(buildJMXMethodExtend(REPLICATION_METHOD_GET_METRIC_FROM_JMX )
|
||||
.jmxObjectName( JMX_LOG_LOG_END_OFFSET ).jmxAttribute(VALUE)));
|
||||
|
||||
// LogStartOffset 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name( REPLICATION_METRIC_LOG_START_OFFSET ).unit("条").desc("副本的LogStartOffset")
|
||||
.extend(buildJMXMethodExtend(REPLICATION_METHOD_GET_METRIC_FROM_JMX )
|
||||
.jmxObjectName( JMX_LOG_LOG_START_OFFSET ).jmxAttribute(VALUE)));
|
||||
|
||||
// Messages 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name( REPLICATION_METRIC_MESSAGES ).unit("条").desc("副本的总消息条数")
|
||||
.extendMethod( REPLICATION_METHOD_GET_METRIC_MESSAGES ));
|
||||
|
||||
// LogSize 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(REPLICATION_METRIC_LOG_SIZE).unit("byte").desc("副本的容量大小")
|
||||
.extend(buildJMXMethodExtend(REPLICATION_METHOD_GET_METRIC_FROM_JMX )
|
||||
.jmxObjectName( JMX_LOG_LOG_SIZE ).jmxAttribute(VALUE)));
|
||||
|
||||
// InSync 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(REPLICATION_METRIC_IN_SYNC).unit("是/否").desc("副本处于ISR中")
|
||||
.extendMethod( REPLICATION_METHOD_GET_IN_SYNC ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(REPLICATION_METRIC_COLLECT_COST_TIME).unit("秒").desc("采集Replica指标的耗时").category(CATEGORY_PERFORMANCE)
|
||||
.extendMethod(REPLICATION_METHOD_DO_NOTHING));
|
||||
|
||||
return itemList;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package com.xiaojukeji.know.streaming.km.core.service.version.metrics;
|
||||
|
||||
import com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem;
|
||||
import com.xiaojukeji.know.streaming.km.common.constant.Constant;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.xiaojukeji.know.streaming.km.common.bean.entity.version.VersionMetricControlItem.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.enums.version.VersionItemTypeEnum.METRIC_TOPIC;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxAttribute.*;
|
||||
import static com.xiaojukeji.know.streaming.km.common.jmx.JmxName.*;
|
||||
import static com.xiaojukeji.know.streaming.km.core.service.topic.impl.TopicMetricServiceImpl.*;
|
||||
|
||||
@Component
|
||||
public class TopicMetricVersionItems extends BaseMetricVersionMetric {
|
||||
|
||||
public static final String TOPIC_METRIC_HEALTH_SCORE = "HealthScore";
|
||||
public static final String TOPIC_METRIC_HEALTH_CHECK_PASSED = "HealthCheckPassed";
|
||||
public static final String TOPIC_METRIC_HEALTH_CHECK_TOTAL = "HealthCheckTotal";
|
||||
public static final String TOPIC_METRIC_TOTAL_PRODUCE_REQUESTS = "TotalProduceRequests";
|
||||
public static final String TOPIC_METRIC_BYTES_REJECTED = "BytesRejected";
|
||||
public static final String TOPIC_METRIC_FAILED_FETCH_REQ = "FailedFetchRequests";
|
||||
public static final String TOPIC_METRIC_FAILED_PRODUCE_REQ = "FailedProduceRequests";
|
||||
public static final String TOPIC_METRIC_REP_COUNT = "ReplicationCount";
|
||||
public static final String TOPIC_METRIC_MESSAGES = "Messages";
|
||||
public static final String TOPIC_METRIC_MESSAGE_IN = "MessagesIn";
|
||||
public static final String TOPIC_METRIC_BYTES_IN = "BytesIn";
|
||||
public static final String TOPIC_METRIC_BYTES_IN_MIN_5 = "BytesIn_min_5";
|
||||
public static final String TOPIC_METRIC_BYTES_IN_MIN_15 = "BytesIn_min_15";
|
||||
public static final String TOPIC_METRIC_BYTES_OUT = "BytesOut";
|
||||
public static final String TOPIC_METRIC_BYTES_OUT_MIN_5 = "BytesOut_min_5";
|
||||
public static final String TOPIC_METRIC_BYTES_OUT_MIN_15 = "BytesOut_min_15";
|
||||
public static final String TOPIC_METRIC_LOG_SIZE = "LogSize";
|
||||
public static final String TOPIC_METRIC_UNDER_REPLICA_PARTITIONS = "PartitionURP";
|
||||
public static final String TOPIC_METRIC_COLLECT_COST_TIME = Constant.COLLECT_METRICS_COST_TIME_METRICS_NAME;
|
||||
|
||||
@Override
|
||||
public int versionItemType() {
|
||||
return METRIC_TOPIC.getCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VersionMetricControlItem> init(){
|
||||
List<VersionMetricControlItem> itemList = new ArrayList<>();
|
||||
|
||||
// HealthScore 指标
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_HEALTH_SCORE).unit("分").desc("健康分").category(CATEGORY_HEALTH)
|
||||
.extendMethod( TOPIC_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name( TOPIC_METRIC_HEALTH_CHECK_PASSED ).unit("个").desc("健康项检查通过数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( TOPIC_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name( TOPIC_METRIC_HEALTH_CHECK_TOTAL ).unit("个").desc("健康项检查总数").category(CATEGORY_HEALTH)
|
||||
.extendMethod( TOPIC_METHOD_GET_HEALTH_SCORE ));
|
||||
|
||||
// TotalProduceRequests 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_TOTAL_PRODUCE_REQUESTS).unit("条/s").desc("Topic 的 TotalProduceRequests").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_PRODUCES_REQUEST ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// BytesRejected 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_REJECTED).unit("个/s").desc("Topic 的每秒写入拒绝量").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTES_REJECTED ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// FailedFetchRequests 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name( TOPIC_METRIC_FAILED_FETCH_REQ ).unit("个/s").desc("Topic 的FailedFetchRequests").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_FAILED_FETCH_REQUEST ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// FailedProduceRequests 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name( TOPIC_METRIC_FAILED_PRODUCE_REQ ).unit("个/s").desc("Topic 的 FailedProduceRequests").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_FAILED_PRODUCE_REQUEST ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// Messages 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_REP_COUNT).unit("个").desc("Topic总的副本数").category(CATEGORY_PERFORMANCE)
|
||||
.extendMethod(TOPIC_METHOD_GET_REPLICAS_COUNT));
|
||||
|
||||
// Messages 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_MESSAGES).unit("条").desc("Topic总的消息数").category(CATEGORY_PERFORMANCE)
|
||||
.extendMethod(TOPIC_METHOD_GET_MESSAGES));
|
||||
|
||||
// MessagesIn 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_MESSAGE_IN).unit("条/s").desc("Topic每秒消息条数").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_MESSAGES_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
// BytesInPerSec 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_IN).unit(BYTE_PER_SEC).desc("Topic每秒消息写入字节数").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_IN_MIN_5).unit(BYTE_PER_SEC).desc("Topic每秒消息写入字节数，5分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_5)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_IN_MIN_15).unit(BYTE_PER_SEC).desc("Topic每秒消息写入字节数，15分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_IN ).jmxAttribute(RATE_MIN_15)));
|
||||
|
||||
// BytesOutPerSec 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_OUT).unit(BYTE_PER_SEC).desc("Topic每秒消息流出字节数").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_1)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_OUT_MIN_5).unit(BYTE_PER_SEC).desc("Topic每秒消息流出字节数，5分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_5)));
|
||||
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_BYTES_OUT_MIN_15).unit(BYTE_PER_SEC).desc("Topic每秒消息流出字节数，15分钟均值").category(CATEGORY_FLOW)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_BROKER_JMX )
|
||||
.jmxObjectName( JMX_SERVER_BROKER_BYTE_OUT ).jmxAttribute(RATE_MIN_15)));
|
||||
|
||||
// LogSize 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_LOG_SIZE).unit("byte").desc("Topic 的大小").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_PARTITION_OF_BROKER_JMX )
|
||||
.jmxObjectName( JMX_LOG_LOG_SIZE ).jmxAttribute(VALUE)));
|
||||
|
||||
// UnderReplicaPartitions 指标
|
||||
itemList.add( buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_UNDER_REPLICA_PARTITIONS).unit("个").desc("Topic未同步的副本数").category(CATEGORY_PERFORMANCE)
|
||||
.extend( buildJMXMethodExtend( TOPIC_METHOD_GET_METRIC_FROM_KAFKA_BY_TOTAL_PARTITION_OF_BROKER_JMX )
|
||||
.jmxObjectName( JMX_CLUSTER_PARTITION_UNDER_REPLICATED ).jmxAttribute(VALUE)));
|
||||
|
||||
itemList.add(buildAllVersionsItem()
|
||||
.name(TOPIC_METRIC_COLLECT_COST_TIME).unit("秒").desc("采集Topic指标的耗时").category(CATEGORY_PERFORMANCE)
|
||||
.extendMethod(TOPIC_METHOD_DO_NOTHING));
|
||||
|
||||
return itemList;
|
||||
}
|
||||
}
|
||||
3
km-core/src/test/java/META-INF/MANIFEST.MF
Normal file
3
km-core/src/test/java/META-INF/MANIFEST.MF
Normal file
@@ -0,0 +1,3 @@
|
||||
Manifest-Version: 1.0
|
||||
Class-Path:
|
||||
|
||||
Reference in New Issue
Block a user