8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

初始化3.0.0版本

Browse Source
This commit is contained in:
zengqiao
2022-08-18 17:04:05 +08:00
parent 462303fca0
commit 51832385b1
2446 changed files with 93177 additions and 127211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
km-extends/km-monitor/pom.xml Normal file
View File

@@ -0,0 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-monitor</artifactId>
<version>${km.revision}</version>
<packaging>jar</packaging>
<parent>
<artifactId>km</artifactId>
<groupId>com.xiaojukeji.kafka</groupId>
<version>${km.revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencies>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-persistence</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.xiaojukeji.kafka</groupId>
<artifactId>km-core</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
</dependencies>
</project>

14
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/common/AbstractMonitorSinkTag.java Normal file
View File

@@ -0,0 +1,14 @@
package com.xiaojukeji.know.streaming.km.monitor.common;
import java.util.Map;
/**
* @author zengqiao
* @date 20/5/24
*/
public abstract class AbstractMonitorSinkTag {
public abstract String convert2Tags();
public abstract Map<String, Object> tagsMap();
}

35
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/common/MetricSinkPoint.java Normal file
View File

@@ -0,0 +1,35 @@
package com.xiaojukeji.know.streaming.km.monitor.common;
import lombok.Data;
import java.util.Map;
@Data
public class MetricSinkPoint {
/**
* 指标名
*/
private String name;
/**
* 指标值
*/
private Float value;
/**
* 上报周期，单位秒
*/
private int step;
/**
* 当前时间戳，单位为s
*/
private long timestamp;
/**
* tags
*/
private Map<String, Object> tagsMap;
public MetricSinkPoint() {}
}

30
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/common/MonitorSinkTagEnum.java Normal file
View File

@@ -0,0 +1,30 @@
package com.xiaojukeji.know.streaming.km.monitor.common;
public enum MonitorSinkTagEnum {
/**
* cluster
*/
CLUSTER_ID("clusterId"),
BROKER_ID("brokerId"),
TOPIC("topic"),
PARTITION_ID("partitionId"),
CONSUMER_GROUP("consumerGroup"),
REPLICATION("replication"),
;
private final String name;
MonitorSinkTagEnum(String name) {
this.name = name;
}
public String getName() {
return name;
}
}

180
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/component/AbstractMonitorSinkService.java Normal file
View File

@@ -0,0 +1,180 @@
package com.xiaojukeji.know.streaming.km.monitor.component;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.*;
import com.xiaojukeji.know.streaming.km.common.bean.event.metric.*;
import com.xiaojukeji.know.streaming.km.common.utils.NamedThreadFactory;
import com.xiaojukeji.know.streaming.km.monitor.common.MetricSinkPoint;
import org.springframework.context.ApplicationListener;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static com.xiaojukeji.know.streaming.km.monitor.common.MonitorSinkTagEnum.*;
public abstract class AbstractMonitorSinkService implements ApplicationListener<BaseMetricEvent> {
protected static final ILog LOGGER = LogFactory.getLog(AbstractMonitorSinkService.class);
private static final int STEP = 60;
private ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 10, 6000, TimeUnit.MILLISECONDS,
new LinkedBlockingDeque<>(1000),
new NamedThreadFactory("KM-Monitor-Sink-" + monitorName()),
(r, e) -> LOGGER.warn("class=AbstractMonitorSinkService||msg=Deque is blocked, taskCount:{}" + e.getTaskCount()));
/**
* monitor 服务的名称
* @return
*/
protected abstract String monitorName();
@Override
public void onApplicationEvent(BaseMetricEvent event) {
executor.execute( () -> {
if(event instanceof BrokerMetricEvent){
BrokerMetricEvent brokerMetricEvent = (BrokerMetricEvent)event;
sinkMetrics(brokerMetric2SinkPoint(brokerMetricEvent.getBrokerMetrics()));
}else if(event instanceof ClusterMetricEvent){
ClusterMetricEvent clusterMetricEvent = (ClusterMetricEvent)event;
sinkMetrics(clusterMetric2SinkPoint(clusterMetricEvent.getClusterMetrics()));
}else if(event instanceof TopicMetricEvent){
TopicMetricEvent topicMetricEvent = (TopicMetricEvent)event;
sinkMetrics(topicMetric2SinkPoint(topicMetricEvent.getTopicMetrics()));
}else if(event instanceof PartitionMetricEvent){
PartitionMetricEvent partitionMetricEvent = (PartitionMetricEvent)event;
sinkMetrics(partitionMetric2SinkPoint(partitionMetricEvent.getPartitionMetrics()));
}else if(event instanceof GroupMetricEvent){
GroupMetricEvent groupMetricEvent = (GroupMetricEvent)event;
sinkMetrics(groupMetric2SinkPoint(groupMetricEvent.getGroupMetrics()));
}else if(event instanceof ReplicaMetricEvent){
ReplicaMetricEvent replicaMetricEvent = (ReplicaMetricEvent)event;
sinkMetrics(replicationMetric2SinkPoint(replicaMetricEvent.getReplicationMetrics()));
}
} );
}
/**
* 监控指标的上报和查询
* @param pointList
* @return
*/
public abstract Boolean sinkMetrics(List<MetricSinkPoint> pointList);
/**************************************************** private method ****************************************************/
private List<MetricSinkPoint> brokerMetric2SinkPoint(List<BrokerMetrics> brokerMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(BrokerMetrics b : brokerMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), b.getClusterPhyId());
tagsMap.put(BROKER_ID.getName(), b.getBrokerId());
pointList.addAll(genSinkPoint("Broker", b.getMetrics(), b.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> clusterMetric2SinkPoint(List<ClusterMetrics> clusterMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(ClusterMetrics c : clusterMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), c.getClusterPhyId());
pointList.addAll(genSinkPoint("Cluster", c.getMetrics(), c.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> topicMetric2SinkPoint(List<TopicMetrics> topicMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(TopicMetrics t : topicMetrics){
if(t.isBBrokerAgg()){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), t.getClusterPhyId());
tagsMap.put(TOPIC.getName(), t.getTopic());
pointList.addAll(genSinkPoint("Topic", t.getMetrics(), t.getTimestamp(), tagsMap));
}
}
return pointList;
}
private List<MetricSinkPoint> partitionMetric2SinkPoint(List<PartitionMetrics> partitionMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(PartitionMetrics p : partitionMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), p.getClusterPhyId());
tagsMap.put(BROKER_ID.getName(), p.getBrokerId());
tagsMap.put(PARTITION_ID.getName(), p.getPartitionId());
pointList.addAll(genSinkPoint("Partition", p.getMetrics(), p.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> groupMetric2SinkPoint(List<GroupMetrics> groupMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(GroupMetrics g : groupMetrics){
if(g.isBGroupMetric()){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), g.getClusterPhyId());
tagsMap.put(CONSUMER_GROUP.getName(), g.getGroup());
pointList.addAll(genSinkPoint("Group", g.getMetrics(), g.getTimestamp(), tagsMap));
}
}
return pointList;
}
private List<MetricSinkPoint> replicationMetric2SinkPoint(List<ReplicationMetrics> replicationMetrics){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(ReplicationMetrics r : replicationMetrics){
Map<String, Object> tagsMap = new HashMap<>();
tagsMap.put(CLUSTER_ID.getName(), r.getClusterPhyId());
tagsMap.put(BROKER_ID.getName(), r.getBrokerId());
tagsMap.put(PARTITION_ID.getName(), r.getPartitionId());
pointList.addAll(genSinkPoint("Replication", r.getMetrics(), r.getTimestamp(), tagsMap));
}
return pointList;
}
private List<MetricSinkPoint> genSinkPoint(String metricPre, Map<String, Float> metrics,
long timeStamp, Map<String, Object> tagsMap){
List<MetricSinkPoint> pointList = new ArrayList<>();
for(String metricName : metrics.keySet()){
MetricSinkPoint metricSinkPoint = new MetricSinkPoint();
metricSinkPoint.setName(metricPre + "_" + metricName);
metricSinkPoint.setValue(metrics.get(metricName));
metricSinkPoint.setTimestamp(timeStamp);
metricSinkPoint.setStep(STEP);
metricSinkPoint.setTagsMap(tagsMap);
pointList.add(metricSinkPoint);
}
return pointList;
}
}

85
km-extends/km-monitor/src/main/java/com/xiaojukeji/know/streaming/km/monitor/component/prometheus/PrometheusService.java Normal file
View File

@@ -0,0 +1,85 @@
package com.xiaojukeji.know.streaming.km.monitor.component.prometheus;
import com.alibaba.fastjson.JSON;
import com.didiglobal.logi.log.ILog;
import com.didiglobal.logi.log.LogFactory;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.monitor.common.MetricSinkPoint;
import com.xiaojukeji.know.streaming.km.monitor.common.MonitorSinkTagEnum;
import com.xiaojukeji.know.streaming.km.monitor.component.AbstractMonitorSinkService;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.Gauge;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author didi
*/
@Service("prometheusService")
public class PrometheusService extends AbstractMonitorSinkService {
private static final ILog LOGGER = LogFactory.getLog(PrometheusService.class);
private final Map<String, Gauge> gaugeMap = new ConcurrentHashMap<>();
private final List<String> monitorSinkTagNames = new ArrayList<>();
@Autowired
public CollectorRegistry collectorRegistry;
@PostConstruct
public void init(){
for(MonitorSinkTagEnum monitorSinkTagEnum : MonitorSinkTagEnum.values()){
monitorSinkTagNames.add(monitorSinkTagEnum.getName());
}
monitorSinkTagNames.sort(Comparator.reverseOrder());
}
@Override
protected String monitorName() {
return "prometheus";
}
@Override
public Boolean sinkMetrics(List<MetricSinkPoint> pointList) {
if(ValidateUtils.isEmptyList(pointList)){
return true;
}
for (MetricSinkPoint metricSinkPoint : pointList){
try {
String metricName = metricSinkPoint.getName().replace("-", "_");
Gauge gauge = gaugeMap.get(metricName);
if(null == gauge) {
gauge = Gauge.build().name(metricName)
.help(metricName + " metrics")
.labelNames(monitorSinkTagNames.toArray(new String[monitorSinkTagNames.size()]))
.register(collectorRegistry);
gaugeMap.put(metricName, gauge);
}
Map<String, Object> tagsMap = metricSinkPoint.getTagsMap();
List<String> labels = new ArrayList<>();
for(String metricTagName : monitorSinkTagNames){
if (tagsMap.containsKey(metricTagName)){
labels.add(tagsMap.get(metricTagName).toString());
} else {
labels.add("");
}
}
gauge.labels(labels.toArray(new String[labels.size()])).set(metricSinkPoint.getValue());
} catch (Exception e){
LOGGER.error("sink metrics to prometheus exception, metric:{}", JSON.toJSONString(metricSinkPoint), e);
}
}
return true;
}
}