8x48/KnowStreaming
1
0
Fork 0
mirror of https://github.com/didi/KnowStreaming.git synced 2025-12-24 11:52:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Optimize]Overview页面的TopN查询ES流程优化(#823)

Browse Source 
1、复用线程池，同时支持线程池的线程数可配置；
2、优化查询TopN指标时，可能会出现重复查询的问题；
3、处理代码扫描(SonarLint)反馈的问题；
This commit is contained in:
zengqiao
2022-12-09 14:34:58 +08:00
committed by EricZeng
parent c4fb18a73c
commit b2f0f69365
11 changed files with 398 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/vo/metrics/point/MetricPointVO.java
View File

@@ -2,7 +2,6 @@ package com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point;
import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@@ -11,7 +10,6 @@ import lombok.NoArgsConstructor;
*/ */
@Data @Data
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor
@ApiModel(description = "指标点") @ApiModel(description = "指标点")
public class MetricPointVO implements Comparable<MetricPointVO> { public class MetricPointVO implements Comparable<MetricPointVO> {
@ApiModelProperty(value = "指标名", example = "HealthScore") @ApiModelProperty(value = "指标名", example = "HealthScore")
@@ -26,6 +24,13 @@ public class MetricPointVO implements Comparable<MetricPointVO> {
@ApiModelProperty(value = "指标值聚合方式：avg、max、min、sum") @ApiModelProperty(value = "指标值聚合方式：avg、max、min、sum")
private String aggType; private String aggType;
public MetricPointVO(String name, Long timeStamp, String value, String aggType) {
this.name = name;
this.timeStamp = timeStamp;
this.value = value;
this.aggType = aggType;
}
@Override @Override
public int compareTo(MetricPointVO o) { public int compareTo(MetricPointVO o) {
if(null == o){return 0;} if(null == o){return 0;}

45
km-core/src/main/java/com/xiaojukeji/know/streaming/km/core/service/version/BaseMetricService.java
View File

@@ -38,40 +38,41 @@ public abstract class BaseMetricService extends BaseKafkaVersionControlService {
protected abstract void initRegisterVCHandler(); protected abstract void initRegisterVCHandler();
protected <T> List<MetricMultiLinesVO> metricMap2VO(Long clusterId, protected <T> List<MetricMultiLinesVO> metricMap2VO(Long clusterId, Map<String/*metric*/, Map<T, List<MetricPointVO>>> metricsMap ){
Map<String/*metric*/, Map<T, List<MetricPointVO>>> map){ List<MetricMultiLinesVO> lineVOList = new ArrayList<>();
List<MetricMultiLinesVO> multiLinesVOS = new ArrayList<>(); if (metricsMap == null || metricsMap.isEmpty()) {
if (map == null || map.isEmpty()) {
// 如果为空，则直接返回 // 如果为空，则直接返回
return multiLinesVOS; return lineVOList;
} }
for(String metric : map.keySet()){ for(Map.Entry<String/*metric*/, Map<T, List<MetricPointVO>>> entry : metricsMap.entrySet()){
try { try {
MetricMultiLinesVO multiLinesVO = new MetricMultiLinesVO(); MetricMultiLinesVO multiLinesVO = new MetricMultiLinesVO();
multiLinesVO.setMetricName(metric); multiLinesVO.setMetricName(entry.getKey());
List<MetricLineVO> metricLines = new ArrayList<>(); if(null == entry.getValue() || entry.getValue().isEmpty()){
continue;
Map<T, List<MetricPointVO>> metricPointMap = map.get(metric);
if(null == metricPointMap || metricPointMap.isEmpty()){continue;}
for(Map.Entry<T, List<MetricPointVO>> entry : metricPointMap.entrySet()){
MetricLineVO metricLineVO = new MetricLineVO();
metricLineVO.setName(entry.getKey().toString());
metricLineVO.setMetricName(metric);
metricLineVO.setMetricPoints(entry.getValue());
metricLines.add(metricLineVO);
} }
List<MetricLineVO> metricLines = new ArrayList<>();
entry.getValue().entrySet().forEach(resNameAndMetricsEntry -> {
MetricLineVO metricLineVO = new MetricLineVO();
metricLineVO.setName(resNameAndMetricsEntry.getKey().toString());
metricLineVO.setMetricName(entry.getKey());
metricLineVO.setMetricPoints(resNameAndMetricsEntry.getValue());
metricLines.add(metricLineVO);
});
multiLinesVO.setMetricLines(metricLines); multiLinesVO.setMetricLines(metricLines);
multiLinesVOS.add(multiLinesVO);
}catch (Exception e){ lineVOList.add(multiLinesVO);
LOGGER.error("method=metricMap2VO||cluster={}||msg=exception!", clusterId, e); } catch (Exception e){
LOGGER.error("method=metricMap2VO||clusterId={}||msg=exception!", clusterId, e);
} }
} }
return multiLinesVOS; return lineVOList;
} }
/** /**

41
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/ESTPService.java Normal file
View File

@@ -0,0 +1,41 @@
package com.xiaojukeji.know.streaming.km.persistence.es;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import lombok.NoArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
/**
* 处理ES请求的线程池
*/
@Service
@NoArgsConstructor
public class ESTPService {
@Value("${thread-pool.es.search.thread-num:10}")
private Integer esSearchThreadCnt;
@Value("${thread-pool.es.search.queue-size:5000}")
private Integer esSearchThreadQueueSize;
private FutureWaitUtil<Object> searchESTP;
@PostConstruct
private void init() {
searchESTP = FutureWaitUtil.init(
"SearchESTP",
esSearchThreadCnt,
esSearchThreadCnt,
esSearchThreadQueueSize
);
}
public void submitSearchTask(String taskName, Integer timeoutUnisMs, Runnable runnable) {
searchESTP.runnableTask(taskName, timeoutUnisMs, runnable);
}
public void waitExecute() {
searchESTP.waitExecute();
}
}

24
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BaseMetricESDAO.java
View File

@@ -12,7 +12,9 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPoint
import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils;
import com.xiaojukeji.know.streaming.km.common.utils.EnvUtil; import com.xiaojukeji.know.streaming.km.common.utils.EnvUtil;
import com.xiaojukeji.know.streaming.km.common.utils.IndexNameUtils; import com.xiaojukeji.know.streaming.km.common.utils.IndexNameUtils;
import com.xiaojukeji.know.streaming.km.common.utils.ValidateUtils;
import com.xiaojukeji.know.streaming.km.persistence.es.BaseESDAO; import com.xiaojukeji.know.streaming.km.persistence.es.BaseESDAO;
import com.xiaojukeji.know.streaming.km.persistence.es.ESTPService;
import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant;
import com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateLoaderUtil; import com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateLoaderUtil;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@@ -48,6 +50,9 @@ public class BaseMetricESDAO extends BaseESDAO {
@Autowired @Autowired
private TemplateLoaderUtil templateLoaderUtil; private TemplateLoaderUtil templateLoaderUtil;
@Autowired
protected ESTPService esTPService;
/** /**
* es 地址 * es 地址
*/ */
@@ -364,21 +369,16 @@ public class BaseMetricESDAO extends BaseESDAO {
sb.append(str, 1, str.length() - 1); sb.append(str, 1, str.length() - 1);
} }
protected Map<String, ESAggr> checkBucketsAndHitsOfResponseAggs(ESQueryResponse response){ protected Map<String, ESAggr> checkBucketsAndHitsOfResponseAggs(ESQueryResponse response) {
if(null == response || null == response.getAggs()){ if(null == response
|| null == response.getAggs()
|| null == response.getAggs().getEsAggrMap()
|| null == response.getAggs().getEsAggrMap().get(HIST)
|| ValidateUtils.isEmptyList(response.getAggs().getEsAggrMap().get(HIST).getBucketList())) {
return null; return null;
} }
Map<String, ESAggr> esAggrMap = response.getAggs().getEsAggrMap(); return response.getAggs().getEsAggrMap();
if (null == esAggrMap || null == esAggrMap.get(HIST)) {
return null;
}
if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){
return null;
}
return esAggrMap;
} }
protected int handleESQueryResponseCount(ESQueryResponse response){ protected int handleESQueryResponseCount(ESQueryResponse response){

120
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BrokerMetricESDAO.java
View File

@@ -6,12 +6,10 @@ import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table; import com.google.common.collect.Table;
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.BrokerMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.BrokerMetricPO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.util.*; import java.util.*;
@@ -29,8 +27,6 @@ public class BrokerMetricESDAO extends BaseMetricESDAO {
register( this); register( this);
} }
protected FutureWaitUtil<Void> queryFuture = FutureWaitUtil.init("BrokerMetricESDAO", 4,8, 500);
/** /**
* 获取集群 clusterId 中 brokerId 最新的统计指标 * 获取集群 clusterId 中 brokerId 最新的统计指标
*/ */
@@ -140,7 +136,7 @@ public class BrokerMetricESDAO extends BaseMetricESDAO {
aggDsl aggDsl
); );
queryFuture.runnableTask( esTPService.submitSearchTask(
String.format("class=BrokerMetricESDAO||method=listBrokerMetricsByBrokerIds||ClusterPhyId=%d", clusterPhyId), String.format("class=BrokerMetricESDAO||method=listBrokerMetricsByBrokerIds||ClusterPhyId=%d", clusterPhyId),
5000, 5000,
() -> { () -> {
@@ -163,7 +159,7 @@ public class BrokerMetricESDAO extends BaseMetricESDAO {
} }
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return table; return table;
} }
@@ -220,106 +216,86 @@ public class BrokerMetricESDAO extends BaseMetricESDAO {
return metricMap; return metricMap;
} }
private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){ private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metricNameList, String aggType){
Map<String, List<MetricPointVO>> metricMap = new HashMap<>(); Map<String, List<MetricPointVO>> metricMap = new HashMap<>();
if(null == response || null == response.getAggs()){ Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if (esAggrMap == null) {
return metricMap; return metricMap;
} }
Map<String, ESAggr> esAggrMap = response.getAggs().getEsAggrMap(); for(String metricName : metricNameList){
if (null == esAggrMap || null == esAggrMap.get(HIST)) {
return metricMap;
}
if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){
return metricMap;
}
for(String metric : metrics){
List<MetricPointVO> metricPoints = new ArrayList<>(); List<MetricPointVO> metricPoints = new ArrayList<>();
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach(esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null == esBucket.getUnusedMap().get(KEY)) {
Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); return;
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE);
if(null == value){return;}
MetricPointVO metricPoint = new MetricPointVO();
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
}else {
LOGGER.info("");
} }
}catch (Exception e){
LOGGER.error("metric={}||errMsg=exception!", metric, e); Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(metricName).getUnusedMap().get(VALUE);
if(null == value) {
return;
}
metricPoints.add(new MetricPointVO(metricName, timestamp, value.toString(), aggType));
} catch (Exception e){
LOGGER.error("method=handleListESQueryResponse||metricName={}||errMsg=exception!", metricName, e);
} }
} ); } );
metricMap.put(metric, optimizeMetricPoints(metricPoints)); metricMap.put(metricName, optimizeMetricPoints(metricPoints));
} }
return metricMap; return metricMap;
} }
private Map<String, List<Long>> handleTopBrokerESQueryResponse(ESQueryResponse response, List<String> metrics, int topN){ private Map<String, List<Long>> handleTopBrokerESQueryResponse(ESQueryResponse response, List<String> metricNameList, int topN) {
Map<String, List<Long>> ret = new HashMap<>(); Map<String, List<Long>> ret = new HashMap<>();
if(null == response || null == response.getAggs()){ Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if (esAggrMap == null) {
return ret; return ret;
} }
Map<String, ESAggr> esAggrMap = response.getAggs().getEsAggrMap(); Map<String, List<Tuple<Long, Double>>> metricNameBrokerValueMap = new HashMap<>();
if (null == esAggrMap || null == esAggrMap.get(HIST)) {
return ret;
}
if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){
return ret;
}
Map<String, List<Tuple<Long, Double>>> metricBrokerValueMap = new HashMap<>();
//1、先获取每个指标对应的所有brokerIds以及指标的值 //1、先获取每个指标对应的所有brokerIds以及指标的值
for(String metric : metrics) { for(String metricName : metricNameList) {
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach(esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null == esBucket.getUnusedMap().get(KEY)) {
Long brokerId = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); return;
Object value = esBucket.getAggrMap().get(HIST).getBucketList().get(0).getAggrMap()
.get(metric).getUnusedMap().get(VALUE);
if(null == value){return;}
List<Tuple<Long, Double>> brokerValue = (null == metricBrokerValueMap.get(metric)) ?
new ArrayList<>() : metricBrokerValueMap.get(metric);
brokerValue.add(new Tuple<>(brokerId, Double.valueOf(value.toString())));
metricBrokerValueMap.put(metric, brokerValue);
} }
}catch (Exception e){
LOGGER.error("metrice={}||errMsg=exception!", metric, e); Long brokerId = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(HIST).getBucketList().get(0).getAggrMap().get(metricName).getUnusedMap().get(VALUE);
if(null == value) {
return;
}
metricNameBrokerValueMap.putIfAbsent(metricName, new ArrayList<>());
metricNameBrokerValueMap.get(metricName).add(new Tuple<>(brokerId, Double.valueOf(value.toString())));
} catch (Exception e) {
LOGGER.error("method=handleTopBrokerESQueryResponse||metric={}||errMsg=exception!", metricName, e);
} }
} ); });
} }
//2、对每个指标的broker按照指标值排序，并截取前topN个brokerIds //2、对每个指标的broker按照指标值排序，并截取前topN个brokerIds
for(String metric : metricBrokerValueMap.keySet()){ for(Map.Entry<String, List<Tuple<Long, Double>>> entry : metricNameBrokerValueMap.entrySet()){
List<Tuple<Long, Double>> brokerValue = metricBrokerValueMap.get(metric); entry.getValue().sort((o1, o2) -> {
if(null == o1 || null == o2){
return 0;
}
brokerValue.sort((o1, o2) -> {
if(null == o1 || null == o2){return 0;}
return o2.getV2().compareTo(o1.getV2()); return o2.getV2().compareTo(o1.getV2());
} ); } );
List<Tuple<Long, Double>> temp = (brokerValue.size() > topN) ? brokerValue.subList(0, topN) : brokerValue; // 获取TopN的Broker
List<Long> brokerIds = temp.stream().map(t -> t.getV1()).collect( Collectors.toList()); List<Long> brokerIdList = entry.getValue().subList(0, Math.min(topN, entry.getValue().size())).stream().map(elem -> elem.getV1()).collect(Collectors.toList());
ret.put(entry.getKey(), brokerIdList);
ret.put(metric, brokerIds);
} }
return ret; return ret;

60
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ClusterMetricESDAO.java
View File

@@ -10,7 +10,6 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchRange;
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchSort; import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchSort;
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ClusterMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ClusterMetricPO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils;
import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -35,8 +34,6 @@ public class ClusterMetricESDAO extends BaseMetricESDAO {
register(this); register(this);
} }
protected FutureWaitUtil<Void> queryFuture = FutureWaitUtil.init("ClusterMetricESDAO", 4,8, 500);
/** /**
* 获取集群 clusterId 最新的统计指标 * 获取集群 clusterId 最新的统计指标
*/ */
@@ -127,30 +124,39 @@ public class ClusterMetricESDAO extends BaseMetricESDAO {
//4、构造dsl查询条件，开始查询 //4、构造dsl查询条件，开始查询
for(Long clusterPhyId : clusterPhyIds){ for(Long clusterPhyId : clusterPhyIds){
try { try {
queryFuture.runnableTask( esTPService.submitSearchTask(
String.format("class=ClusterMetricESDAO||method=listClusterMetricsByClusterIds||ClusterPhyId=%d", clusterPhyId), String.format("class=ClusterMetricESDAO||method=listClusterMetricsByClusterIds||ClusterPhyId=%d", clusterPhyId),
5000, 5000,
() -> { () -> {
String dsl = dslLoaderUtil.getFormatDslByFileName( String dsl = dslLoaderUtil.getFormatDslByFileName(
DslConstant.GET_CLUSTER_AGG_LIST_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); DslConstant.GET_CLUSTER_AGG_LIST_METRICS,
clusterPhyId,
startTime,
endTime,
interval,
aggDsl
);
Map<String/*metric*/, List<MetricPointVO>> metricMap = esOpClient.performRequestWithRouting( Map<String/*metric*/, List<MetricPointVO>> metricMap = esOpClient.performRequestWithRouting(
String.valueOf(clusterPhyId), realIndex, dsl, String.valueOf(clusterPhyId),
s -> handleListESQueryResponse(s, metrics, aggType), 3); realIndex,
dsl,
s -> handleListESQueryResponse(s, metrics, aggType),
DEFAULT_RETRY_TIME
);
synchronized (table){ synchronized (table){
for(String metric : metricMap.keySet()){ for(Map.Entry<String/*metric*/, List<MetricPointVO>> entry : metricMap.entrySet()){
table.put(metric, clusterPhyId, metricMap.get(metric)); table.put(entry.getKey(), clusterPhyId, entry.getValue());
} }
} }
}); });
}catch (Exception e){ }catch (Exception e){
LOGGER.error("method=listClusterMetricsByClusterIds||clusterPhyId={}||errMsg=exception!", LOGGER.error("method=listClusterMetricsByClusterIds||clusterPhyId={}||errMsg=exception!", clusterPhyId, e);
clusterPhyId, e);
} }
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return table; return table;
} }
@@ -182,35 +188,33 @@ public class ClusterMetricESDAO extends BaseMetricESDAO {
return metricMap; return metricMap;
} }
private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){ private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metricNameList, String aggType){
Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response); Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap){return new HashMap<>();} if(null == esAggrMap) {
return new HashMap<>();
}
Map<String, List<MetricPointVO>> metricMap = new HashMap<>(); Map<String, List<MetricPointVO>> metricMap = new HashMap<>();
for(String metric : metrics){ for(String metricName : metricNameList) {
List<MetricPointVO> metricPoints = new ArrayList<>(); List<MetricPointVO> metricPoints = new ArrayList<>();
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach( esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null != esBucket.getUnusedMap().get(KEY)) {
Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE); Object value = esBucket.getAggrMap().get(metricName).getUnusedMap().get(VALUE);
if(null == value){return;} if(null == value) {
return;
}
MetricPointVO metricPoint = new MetricPointVO(); metricPoints.add(new MetricPointVO(metricName, timestamp, value.toString(), aggType));
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
} }
}catch (Exception e){ } catch (Exception e){
LOGGER.error("method=handleESQueryResponse||metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleListESQueryResponse||metricName={}||errMsg=exception!", metricName, e);
} }
} ); } );
metricMap.put(metric, optimizeMetricPoints(metricPoints)); metricMap.put(metricName, optimizeMetricPoints(metricPoints));
} }
return metricMap; return metricMap;

62
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/GroupMetricESDAO.java
View File

@@ -10,7 +10,6 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.topic.TopicPartitionK
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.GroupMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.GroupMetricPO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum; import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils;
import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -35,8 +34,6 @@ public class GroupMetricESDAO extends BaseMetricESDAO {
register(this); register(this);
} }
protected FutureWaitUtil<Void> queryFuture = FutureWaitUtil.init("GroupMetricESDAO", 4,8, 500);
public List<GroupMetricPO> listLatestMetricsAggByGroupTopic(Long clusterPhyId, List<GroupTopic> groupTopicList, List<String> metrics, AggTypeEnum aggType){ public List<GroupMetricPO> listLatestMetricsAggByGroupTopic(Long clusterPhyId, List<GroupTopic> groupTopicList, List<String> metrics, AggTypeEnum aggType){
Long latestTime = getLatestMetricTime(); Long latestTime = getLatestMetricTime();
Long startTime = latestTime - FIVE_MIN; Long startTime = latestTime - FIVE_MIN;
@@ -49,7 +46,7 @@ public class GroupMetricESDAO extends BaseMetricESDAO {
List<GroupMetricPO> groupMetricPOS = new CopyOnWriteArrayList<>(); List<GroupMetricPO> groupMetricPOS = new CopyOnWriteArrayList<>();
for(GroupTopic groupTopic : groupTopicList){ for(GroupTopic groupTopic : groupTopicList){
queryFuture.runnableTask( esTPService.submitSearchTask(
String.format("class=GroupMetricESDAO||method=listLatestMetricsAggByGroupTopic||ClusterPhyId=%d||groupName=%s||topicName=%s", String.format("class=GroupMetricESDAO||method=listLatestMetricsAggByGroupTopic||ClusterPhyId=%d||groupName=%s||topicName=%s",
clusterPhyId, groupTopic.getGroupName(), groupTopic.getTopicName()), clusterPhyId, groupTopic.getGroupName(), groupTopic.getTopicName()),
5000, 5000,
@@ -73,7 +70,7 @@ public class GroupMetricESDAO extends BaseMetricESDAO {
}); });
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return groupMetricPOS; return groupMetricPOS;
} }
@@ -126,13 +123,26 @@ public class GroupMetricESDAO extends BaseMetricESDAO {
Integer partition = tp.getPartition(); Integer partition = tp.getPartition();
String dsl = dslLoaderUtil.getFormatDslByFileName( String dsl = dslLoaderUtil.getFormatDslByFileName(
DslConstant.LIST_GROUP_METRICS, clusterId, groupName, topic, partition, startTime, endTime, interval, aggDsl); DslConstant.LIST_GROUP_METRICS,
clusterId,
groupName,
topic,
partition,
startTime,
endTime,
interval,
aggDsl
);
Map<String/*metric*/, List<MetricPointVO>> metricMap = esOpClient.performRequest(realIndex, dsl, Map<String/*metric*/, List<MetricPointVO>> metricMap = esOpClient.performRequest(
s -> handleGroupMetrics(s, aggType, metrics), 3); realIndex,
dsl,
s -> handleGroupMetrics(s, aggType, metrics),
DEFAULT_RETRY_TIME
);
for(String metric : metricMap.keySet()){ for(Map.Entry<String/*metric*/, List<MetricPointVO>> entry: metricMap.entrySet()){
table.put(metric, topic + "&" + partition, metricMap.get(metric)); table.put(entry.getKey(), topic + "&" + partition, entry.getValue());
} }
} }
@@ -188,23 +198,27 @@ public class GroupMetricESDAO extends BaseMetricESDAO {
for(String metric : metrics){ for(String metric : metrics){
List<MetricPointVO> metricPoints = new ArrayList<>(); List<MetricPointVO> metricPoints = new ArrayList<>();
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach(esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null == esBucket.getUnusedMap().get(KEY)) {
Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); return;
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE);
if(value == null){return;}
MetricPointVO metricPoint = new MetricPointVO();
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
} }
Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE);
if(value == null) {
return;
}
MetricPointVO metricPoint = new MetricPointVO();
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
}catch (Exception e){ }catch (Exception e){
LOGGER.error("method=handleESQueryResponse||metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleGroupMetrics||metric={}||errMsg=exception!", metric, e);
} }
} ); } );

157
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/TopicMetricESDAO.java
View File

@@ -10,7 +10,6 @@ import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm;
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchSort; import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchSort;
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.TopicMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.TopicMetricPO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant;
@@ -26,7 +25,6 @@ import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateC
@Component @Component
public class TopicMetricESDAO extends BaseMetricESDAO { public class TopicMetricESDAO extends BaseMetricESDAO {
@PostConstruct @PostConstruct
public void init() { public void init() {
super.indexName = TOPIC_INDEX; super.indexName = TOPIC_INDEX;
@@ -34,8 +32,6 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
register(this); register(this);
} }
protected FutureWaitUtil<Void> queryFuture = FutureWaitUtil.init("TopicMetricESDAO", 4,8, 500);
public List<TopicMetricPO> listTopicMaxMinMetrics(Long clusterPhyId, List<String> topics, String metric, boolean max, Long startTime, Long endTime){ public List<TopicMetricPO> listTopicMaxMinMetrics(Long clusterPhyId, List<String> topics, String metric, boolean max, Long startTime, Long endTime){
//1、获取需要查下的索引 //1、获取需要查下的索引
String realIndex = realIndex(startTime, endTime); String realIndex = realIndex(startTime, endTime);
@@ -128,8 +124,13 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
? dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_TOPIC_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl) ? dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_TOPIC_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl)
: dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_TOPIC_NOT_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl); : dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_TOPIC_NOT_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl);
return esOpClient.performRequestWithRouting(topic, realIndex, dsl, return esOpClient.performRequestWithRouting(
s -> handleESQueryResponseCount(s), 3); topic,
realIndex,
dsl,
s -> handleESQueryResponseCount(s),
DEFAULT_RETRY_TIME
);
} }
/** /**
@@ -207,38 +208,52 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
* 获取每个 metric 的 topN 个 topic 的指标，如果获取不到 topN 的topics, 则默认返回 defaultTopics 的指标 * 获取每个 metric 的 topN 个 topic 的指标，如果获取不到 topN 的topics, 则默认返回 defaultTopics 的指标
*/ */
public Table<String/*metric*/, String/*topics*/, List<MetricPointVO>> listTopicMetricsByTopN(Long clusterPhyId, public Table<String/*metric*/, String/*topics*/, List<MetricPointVO>> listTopicMetricsByTopN(Long clusterPhyId,
List<String> defaultTopics, List<String> defaultTopicNameList,
List<String> metrics, List<String> metricNameList,
String aggType, String aggType,
int topN, int topN,
Long startTime, Long startTime,
Long endTime){ Long endTime){
//1、获取topN要查询的topic，每一个指标的topN的topic可能不一样 //1、获取topN要查询的topic，每一个指标的topN的topic可能不一样
Map<String, List<String>> metricTopics = this.getTopNTopics(clusterPhyId, metrics, aggType, topN, startTime, endTime); Map<String, List<String>> metricTopicsMap = this.getTopNTopics(clusterPhyId, metricNameList, aggType, topN, startTime, endTime);
Table<String, String, List<MetricPointVO>> table = HashBasedTable.create(); //2、获取topics列表
Set<String> topicNameSet = new HashSet<>(defaultTopicNameList);
metricTopicsMap.values().forEach(elem -> topicNameSet.addAll(elem));
for(String metric : metrics) { //3、批量获取信息
table.putAll(this.listTopicMetricsByTopics( Table<String, String, List<MetricPointVO>> allMetricsTable = this.listTopicMetricsByTopics(
clusterPhyId, clusterPhyId,
Arrays.asList(metric), metricNameList,
aggType, aggType,
metricTopics.getOrDefault(metric, defaultTopics), new ArrayList<>(topicNameSet),
startTime, startTime,
endTime) endTime
); );
//4、获取Top-Metric
Table<String, String, List<MetricPointVO>> metricsTable = HashBasedTable.create();
for(String metricName: metricNameList) {
for (String topicName: metricTopicsMap.getOrDefault(metricName, defaultTopicNameList)) {
List<MetricPointVO> voList = allMetricsTable.get(metricName, topicName);
if (voList == null) {
continue;
}
metricsTable.put(metricName, topicName, voList);
}
} }
return table; return metricsTable;
} }
/** /**
* 获取每个 metric 指定个 topic 的指标 * 获取每个 metric 指定个 topic 的指标
*/ */
public Table<String/*metric*/, String/*topics*/, List<MetricPointVO>> listTopicMetricsByTopics(Long clusterPhyId, public Table<String/*metric*/, String/*topics*/, List<MetricPointVO>> listTopicMetricsByTopics(Long clusterPhyId,
List<String> metrics, List<String> metricNameList,
String aggType, String aggType,
List<String> topics, List<String> topicNameList,
Long startTime, Long startTime,
Long endTime){ Long endTime){
//1、获取需要查下的索引 //1、获取需要查下的索引
@@ -248,37 +263,47 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
String interval = MetricsUtils.getInterval(endTime - startTime); String interval = MetricsUtils.getInterval(endTime - startTime);
//3、构造agg查询条件 //3、构造agg查询条件
String aggDsl = buildAggsDSL(metrics, aggType); String aggDsl = buildAggsDSL(metricNameList, aggType);
final Table<String, String, List<MetricPointVO>> table = HashBasedTable.create(); final Table<String, String, List<MetricPointVO>> table = HashBasedTable.create();
//4、构造dsl查询条件 //4、构造dsl查询条件
for(String topic : topics){ for(String topicName : topicNameList){
try { try {
queryFuture.runnableTask( esTPService.submitSearchTask(
String.format("class=TopicMetricESDAO||method=listTopicMetricsByTopics||ClusterPhyId=%d||topicName=%s", String.format("class=TopicMetricESDAO||method=listTopicMetricsByTopics||ClusterPhyId=%d||topicName=%s", clusterPhyId, topicName),
clusterPhyId, topic),
3000, 3000,
() -> { () -> {
String dsl = dslLoaderUtil.getFormatDslByFileName( String dsl = dslLoaderUtil.getFormatDslByFileName(
DslConstant.GET_TOPIC_AGG_LIST_METRICS, clusterPhyId, topic, startTime, endTime, interval, aggDsl); DslConstant.GET_TOPIC_AGG_LIST_METRICS,
clusterPhyId,
topicName,
startTime,
endTime,
interval,
aggDsl
);
Map<String/*metric*/, List<MetricPointVO>> metricMap = esOpClient.performRequestWithRouting(topic, realIndex, dsl, Map<String/*metric*/, List<MetricPointVO>> metricMap = esOpClient.performRequestWithRouting(
s -> handleListESQueryResponse(s, metrics, aggType), 3); topicName,
realIndex,
dsl,
s -> handleListESQueryResponse(s, metricNameList, aggType),
DEFAULT_RETRY_TIME
);
synchronized (table){ synchronized (table){
for(String metric : metricMap.keySet()){ for(Map.Entry<String/*metric*/, List<MetricPointVO>> entry: metricMap.entrySet()){
table.put(metric, topic, metricMap.get(metric)); table.put(entry.getKey(), topicName, entry.getValue());
} }
} }
}); });
}catch (Exception e){ }catch (Exception e){
LOGGER.error("method=listBrokerMetricsByBrokerIds||clusterPhyId={}||brokerId{}||errMsg=exception!", LOGGER.error("method=listTopicMetricsByTopics||clusterPhyId={}||topicName={}||errMsg=exception!", clusterPhyId, topicName, e);
clusterPhyId, topic, e);
} }
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return table; return table;
} }
@@ -339,7 +364,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){ private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){
Map<String, List<MetricPointVO>> metricMap = new HashMap<>(); Map<String, List<MetricPointVO>> metricMap = new HashMap<>();
Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response); Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap){return metricMap;} if(null == esAggrMap){return metricMap;}
for(String metric : metrics){ for(String metric : metrics){
@@ -352,15 +377,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE); Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE);
if(value == null){return;} if(value == null){return;}
MetricPointVO metricPoint = new MetricPointVO(); metricPoints.add(new MetricPointVO(metric, timestamp, value.toString(), aggType));
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
}else {
LOGGER.info("");
} }
}catch (Exception e){ }catch (Exception e){
LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e);
@@ -373,7 +390,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
return metricMap; return metricMap;
} }
private Map<String, List<String>> handleTopTopicESQueryResponse(ESQueryResponse response, List<String> metrics, int topN){ private Map<String, List<String>> handleTopTopicESQueryResponse(ESQueryResponse response, List<String> metricNameList, int topN){
Map<String, List<String>> ret = new HashMap<>(); Map<String, List<String>> ret = new HashMap<>();
Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response); Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response);
@@ -382,57 +399,37 @@ public class TopicMetricESDAO extends BaseMetricESDAO {
Map<String, List<Tuple<String, Double>>> metricsTopicValueMap = new HashMap<>(); Map<String, List<Tuple<String, Double>>> metricsTopicValueMap = new HashMap<>();
//1、先获取每个指标对应的所有 topic 以及指标的值 //1、先获取每个指标对应的所有 topic 以及指标的值
for(String metric : metrics) { for(String metricName: metricNameList) {
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach( esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null != esBucket.getUnusedMap().get(KEY)) {
String topic = esBucket.getUnusedMap().get(KEY).toString(); String topic = esBucket.getUnusedMap().get(KEY).toString();
Double value = Double.valueOf(esBucket.getAggrMap().get(HIST).getBucketList().get(0).getAggrMap() Double value = Double.valueOf(
.get(metric).getUnusedMap().get(VALUE).toString()); esBucket.getAggrMap().get(HIST).getBucketList().get(0).getAggrMap().get(metricName).getUnusedMap().get(VALUE).toString()
);
List<Tuple<String, Double>> brokerValue = (null == metricsTopicValueMap.get(metric)) ? metricsTopicValueMap.putIfAbsent(metricName, new ArrayList<>());
new ArrayList<>() : metricsTopicValueMap.get(metric); metricsTopicValueMap.get(metricName).add(new Tuple<>(topic, value));
brokerValue.add(new Tuple<>(topic, value));
metricsTopicValueMap.put(metric, brokerValue);
} }
}catch (Exception e){ }catch (Exception e){
LOGGER.error("method=handleTopBrokerESQueryResponse||metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleTopTopicESQueryResponse||metricName={}||errMsg=exception!", metricName, e);
} }
} ); } );
} }
//2、对每个指标的broker按照指标值排序，并截取前topN个brokerIds //2、对每个指标的broker按照指标值排序，并截取前topN个brokerIds
for(String metric : metricsTopicValueMap.keySet()){ for(Map.Entry<String, List<Tuple<String, Double>>> entry: metricsTopicValueMap.entrySet()){
List<Tuple<String, Double>> brokerValue = metricsTopicValueMap.get(metric); entry.getValue().sort((o1, o2) -> {
if(null == o1 || null == o2) {
return 0;
}
brokerValue.sort((o1, o2) -> {
if(null == o1 || null == o2){return 0;}
return o2.getV2().compareTo(o1.getV2()); return o2.getV2().compareTo(o1.getV2());
} ); } );
List<Tuple<String, Double>> temp = (brokerValue.size() > topN) ? brokerValue.subList(0, topN) : brokerValue; List<String> topicNameList = entry.getValue().subList(0, Math.min(entry.getValue().size(), topN)).stream().map(t -> t.getV1()).collect(Collectors.toList());
List<String> topics = temp.stream().map(t -> t.getV1()).collect(Collectors.toList());
ret.put(metric, topics); ret.put(entry.getKey(), topicNameList);
}
return ret;
}
private Map<String/*metric*/, Map<String/*topic*/, List<MetricPointVO>>> topicMetricMap2MetricTopicMap(
Map<String/*topic*/, Map<String/*metric*/, List<MetricPointVO>>> topicMetricMap){
Map<String/*metric*/, Map<String/*topic*/, List<MetricPointVO>>> ret = new HashMap<>();
for(String topic : topicMetricMap.keySet()){
Map<String/*metric*/, List<MetricPointVO>> metricMap = topicMetricMap.get(topic);
for(String metric : metricMap.keySet()){
Map<String/*topic*/, List<MetricPointVO>> brokerMap = (null == ret.get(metric)) ? new HashMap<>() : ret.get(metric);
brokerMap.put(topic, metricMap.get(metric));
ret.put(metric, brokerMap);
}
} }
return ret; return ret;

122
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectClusterMetricESDAO.java
View File

@@ -5,13 +5,11 @@ import com.didiglobal.logi.elasticsearch.client.response.query.query.aggs.ESAggr
import com.google.common.collect.HashBasedTable; import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table; import com.google.common.collect.Table;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
import com.xiaojukeji.know.streaming.km.persistence.es.dao.BaseMetricESDAO; import com.xiaojukeji.know.streaming.km.persistence.es.dao.BaseMetricESDAO;
import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.util.*; import java.util.*;
@@ -29,8 +27,6 @@ public class ConnectClusterMetricESDAO extends BaseMetricESDAO {
register( this); register( this);
} }
protected FutureWaitUtil<Void> queryFuture = FutureWaitUtil.init("ConnectClusterMetricESDAO", 4,8, 500);
/** /**
* 获取集群 clusterPhyId 中每个 metric 的 topN 的 connectCluster 在指定时间[startTime、endTime]区间内所有的指标 * 获取集群 clusterPhyId 中每个 metric 的 topN 的 connectCluster 在指定时间[startTime、endTime]区间内所有的指标
* topN 按照[startTime, endTime] 时间段内最后一个值来排序 * topN 按照[startTime, endTime] 时间段内最后一个值来排序
@@ -97,7 +93,7 @@ public class ConnectClusterMetricESDAO extends BaseMetricESDAO {
aggDsl aggDsl
); );
queryFuture.runnableTask( esTPService.submitSearchTask(
String.format("class=ConnectClusterMetricESDAO||method=listMetricsByConnectClusterIdList||ClusterPhyId=%d", clusterPhyId), String.format("class=ConnectClusterMetricESDAO||method=listMetricsByConnectClusterIdList||ClusterPhyId=%d", clusterPhyId),
5000, 5000,
() -> { () -> {
@@ -106,24 +102,24 @@ public class ConnectClusterMetricESDAO extends BaseMetricESDAO {
realIndex, realIndex,
dsl, dsl,
s -> handleListESQueryResponse(s, metricNameList, aggType), s -> handleListESQueryResponse(s, metricNameList, aggType),
3 DEFAULT_RETRY_TIME
); );
synchronized (table) { synchronized (table) {
for(String metric : metricMap.keySet()){ for(Map.Entry<String, List<MetricPointVO>> entry : metricMap.entrySet()){
table.put(metric, connectClusterId, metricMap.get(metric)); table.put(entry.getKey(), connectClusterId, entry.getValue());
} }
} }
}); });
} catch (Exception e) { } catch (Exception e) {
LOGGER.error( LOGGER.error(
"class=ConnectClusterMetricESDAO||method=listMetricsByConnectClusterIdList||clusterPhyId={}||connectClusterId{}||errMsg=exception!", "method=listMetricsByConnectClusterIdList||clusterPhyId={}||connectClusterId={}||errMsg=exception!",
clusterPhyId, connectClusterId, e clusterPhyId, connectClusterId, e
); );
} }
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return table; return table;
} }
@@ -167,107 +163,89 @@ public class ConnectClusterMetricESDAO extends BaseMetricESDAO {
/**************************************************** private method ****************************************************/ /**************************************************** private method ****************************************************/
private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){ private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metricNameList, String aggType){
Map<String, List<MetricPointVO>> metricMap = new HashMap<>(); Map<String, List<MetricPointVO>> metricMap = new HashMap<>();
if(null == response || null == response.getAggs()){ Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap) {
return metricMap; return metricMap;
} }
Map<String, ESAggr> esAggrMap = response.getAggs().getEsAggrMap(); for(String metricName : metricNameList){
if (null == esAggrMap || null == esAggrMap.get(HIST)) {
return metricMap;
}
if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){
return metricMap;
}
for(String metric : metrics){
List<MetricPointVO> metricPoints = new ArrayList<>(); List<MetricPointVO> metricPoints = new ArrayList<>();
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach(esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null != esBucket.getUnusedMap().get(KEY)) {
Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE); Object value = esBucket.getAggrMap().get(metricName).getUnusedMap().get(VALUE);
if(null == value){return;} if(null == value) {
return;
}
MetricPointVO metricPoint = new MetricPointVO(); metricPoints.add(new MetricPointVO(metricName, timestamp, value.toString(), aggType));
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
}else {
LOGGER.info("");
} }
}catch (Exception e){ }catch (Exception e){
LOGGER.error("metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleListESQueryResponse||metricName={}||errMsg=exception!", metricName, e);
} }
} ); } );
metricMap.put(metric, optimizeMetricPoints(metricPoints)); metricMap.put(metricName, optimizeMetricPoints(metricPoints));
} }
return metricMap; return metricMap;
} }
private Map<String, List<Long>> handleTopConnectClusterESQueryResponse(ESQueryResponse response, List<String> metrics, int topN){ private Map<String, List<Long>> handleTopConnectClusterESQueryResponse(ESQueryResponse response, List<String> metricNameList, int topN){
Map<String, List<Long>> ret = new HashMap<>(); Map<String, List<Long>> ret = new HashMap<>();
if(null == response || null == response.getAggs()){ Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap) {
return ret; return ret;
} }
Map<String, ESAggr> esAggrMap = response.getAggs().getEsAggrMap(); Map<String, List<Tuple<Long, Double>>> metricConnectClusterValueMap = new HashMap<>();
if (null == esAggrMap || null == esAggrMap.get(HIST)) {
return ret;
}
if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){
return ret;
}
Map<String, List<Tuple<Long, Double>>> metricBrokerValueMap = new HashMap<>();
//1、先获取每个指标对应的所有brokerIds以及指标的值 //1、先获取每个指标对应的所有brokerIds以及指标的值
for(String metric : metrics) { for(String metricName : metricNameList) {
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach(esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null == esBucket.getUnusedMap().get(KEY)) {
Long connectorClusterId = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); return;
Object value = esBucket.getAggrMap().get(HIST).getBucketList()
.get(0).getAggrMap().get(metric).getUnusedMap().get(VALUE);
if(null == value){return;}
List<Tuple<Long, Double>> connectorClusterValue = (null == metricBrokerValueMap.get(metric)) ?
new ArrayList<>() : metricBrokerValueMap.get(metric);
connectorClusterValue.add(new Tuple<>(connectorClusterId, Double.valueOf(value.toString())));
metricBrokerValueMap.put(metric, connectorClusterValue);
} }
Long connectorClusterId = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(HIST).getBucketList().get(0).getAggrMap().get(metricName).getUnusedMap().get(VALUE);
if(null == value) {
return;
}
metricConnectClusterValueMap.putIfAbsent(metricName, new ArrayList<>());
metricConnectClusterValueMap.get(metricName).add(new Tuple<>(connectorClusterId, Double.valueOf(value.toString())));
}catch (Exception e){ }catch (Exception e){
LOGGER.error("metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleTopConnectClusterESQueryResponse||metricName={}||errMsg=exception!", metricName, e);
} }
} ); } );
} }
//2、对每个指标的broker按照指标值排序，并截取前topN个brokerIds //2、对每个指标的connect按照指标值排序，并截取前topN个connectIds
for(String metric : metricBrokerValueMap.keySet()){ for(Map.Entry<String, List<Tuple<Long, Double>>> entry : metricConnectClusterValueMap.entrySet()){
List<Tuple<Long, Double>> connectorClusterValue = metricBrokerValueMap.get(metric);
entry.getValue().sort((o1, o2) -> {
if(null == o1 || null == o2) {
return 0;
}
connectorClusterValue.sort((o1, o2) -> {
if(null == o1 || null == o2){return 0;}
return o2.getV2().compareTo(o1.getV2()); return o2.getV2().compareTo(o1.getV2());
} ); });
List<Tuple<Long, Double>> temp = (connectorClusterValue.size() > topN) ? connectorClusterValue.subList(0, topN) : connectorClusterValue; List<Long> connectorClusterIdList = entry.getValue()
List<Long> connectorClusterIds = temp.stream().map(t -> t.getV1()).collect(Collectors.toList()); .subList(0, Math.min(entry.getValue().size(), topN))
.stream()
.map(t -> t.getV1())
.collect(Collectors.toList());
ret.put(metric, connectorClusterIds); ret.put(entry.getKey(), connectorClusterIdList);
} }
return ret; return ret;

172
km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectorMetricESDAO.java
View File

@@ -8,7 +8,6 @@ import com.google.common.collect.Table;
import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm; import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm;
import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectorMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectorMetricPO;
import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO;
import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil;
import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils;
import com.xiaojukeji.know.streaming.km.common.utils.Triple; import com.xiaojukeji.know.streaming.km.common.utils.Triple;
import com.xiaojukeji.know.streaming.km.common.utils.Tuple; import com.xiaojukeji.know.streaming.km.common.utils.Tuple;
@@ -33,43 +32,55 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
register( this); register( this);
} }
protected FutureWaitUtil<Void> queryFuture = FutureWaitUtil.init("ConnectorMetricESDAO", 4,8, 500);
/** /**
* 获取每个 metric 的 topN 个 connector 的指标，如果获取不到 topN 的topics, 则默认返回 defaultTopics 的指标 * 获取每个 metric 的 topN 个 connector 的指标，如果获取不到 topN 的 connectors, 则默认返回 defaultTopics 的指标
*/ */
public Table<String/*metric*/, Tuple<Long, String>, List<MetricPointVO>> listMetricsByTopN(Long clusterPhyId, public Table<String/*metric*/, Tuple<Long, String>, List<MetricPointVO>> listMetricsByTopN(Long clusterPhyId,
List<Tuple<Long, String>> defaultConnectorList, List<Tuple<Long, String>> defaultConnectorList,
List<String> metricNameList, List<String> metricNameList,
String aggType, String aggType,
int topN, int topN,
Long startTime, Long startTime,
Long endTime){ Long endTime) {
//1、获取topN要查询的topic，每一个指标的topN的topic可能不一样 //1、获取topN要查询的topic，每一个指标的topN的topic可能不一样
Map<String, List<Tuple<Long, String>>> metricsMap = this.getTopNConnectors(clusterPhyId, metricNameList, aggType, topN, startTime, endTime); Map<String, List<Tuple<Long, String>>> metricsMap = this.getTopNConnectors(clusterPhyId, metricNameList, aggType, topN, startTime, endTime);
Table<String, Tuple<Long, String>, List<MetricPointVO>> table = HashBasedTable.create(); //2、获取connector列表
Set<Tuple<Long, String>> connectorSet = new HashSet<>(defaultConnectorList);
metricsMap.values().forEach(elem -> connectorSet.addAll(elem));
for(String metricName : metricNameList){ //3、批量获取信息
table.putAll(this.listMetricsByConnectors( Table<String, Tuple<Long, String>, List<MetricPointVO>> allMetricsTable = this.listMetricsByConnectors(
clusterPhyId, clusterPhyId,
Arrays.asList(metricName), metricNameList,
aggType, aggType,
metricsMap.getOrDefault(metricName, defaultConnectorList), new ArrayList<>(connectorSet),
startTime, startTime,
endTime) endTime
); );
//4、获取Top-Metric
Table<String, Tuple<Long, String>, List<MetricPointVO>> metricTable = HashBasedTable.create();
for (String metricName: metricNameList) {
for (Tuple<Long, String> connector: metricsMap.getOrDefault(metricName, defaultConnectorList)) {
List<MetricPointVO> voList = allMetricsTable.get(metricName, connector);
if (voList == null) {
continue;
}
metricTable.put(metricName, connector, voList);
}
} }
return table; // 返回结果
return metricTable;
} }
public List<ConnectorMetricPO> getConnectorLatestMetric(Long clusterPhyId, List<Tuple<Long, String>> connectClusterIdAndConnectorNameList, List<String> metricsNames){ public List<ConnectorMetricPO> getConnectorLatestMetric(Long clusterPhyId, List<Tuple<Long, String>> connectClusterIdAndConnectorNameList, List<String> metricsNames){
List<ConnectorMetricPO> connectorMetricPOS = new CopyOnWriteArrayList<>(); List<ConnectorMetricPO> connectorMetricPOS = new CopyOnWriteArrayList<>();
for(Tuple<Long, String> connectClusterIdAndConnectorName : connectClusterIdAndConnectorNameList){ for(Tuple<Long, String> connectClusterIdAndConnectorName : connectClusterIdAndConnectorNameList){
queryFuture.runnableTask( esTPService.submitSearchTask(
"getConnectorLatestMetric", "getConnectorLatestMetric",
30000, 30000,
() -> { () -> {
@@ -78,7 +89,7 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
}); });
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return connectorMetricPOS; return connectorMetricPOS;
} }
@@ -112,11 +123,11 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
* 获取每个 metric 指定个 topic 的指标 * 获取每个 metric 指定个 topic 的指标
*/ */
public Table<String/*metric*/, Tuple<Long, String>, List<MetricPointVO>> listMetricsByConnectors(Long clusterPhyId, public Table<String/*metric*/, Tuple<Long, String>, List<MetricPointVO>> listMetricsByConnectors(Long clusterPhyId,
List<String> metrics, List<String> metricNameList,
String aggType, String aggType,
List<Tuple<Long, String>> connectorList, List<Tuple<Long, String>> connectorList,
Long startTime, Long startTime,
Long endTime) { Long endTime) {
//1、获取需要查下的索引 //1、获取需要查下的索引
String realIndex = realIndex(startTime, endTime); String realIndex = realIndex(startTime, endTime);
@@ -124,17 +135,15 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
String interval = MetricsUtils.getInterval(endTime - startTime); String interval = MetricsUtils.getInterval(endTime - startTime);
//3、构造agg查询条件 //3、构造agg查询条件
String aggDsl = buildAggsDSL(metrics, aggType); String aggDsl = buildAggsDSL(metricNameList, aggType);
final Table<String, Tuple<Long, String>, List<MetricPointVO>> table = HashBasedTable.create(); final Table<String, Tuple<Long, String>, List<MetricPointVO>> table = HashBasedTable.create();
//4、构造dsl查询条件 //4、构造dsl查询条件
for(Tuple<Long, String> connector : connectorList) { for(Tuple<Long, String> connector : connectorList) {
try { try {
queryFuture.runnableTask( esTPService.submitSearchTask(
String.format( String.format("class=ConnectorMetricESDAO||method=listMetricsByConnectors||ClusterPhyId=%d||connectorName=%s", clusterPhyId, connector.getV2()),
"method=listConnectorMetricsByConnectors||ClusterPhyId=%d||connectorName=%s",
clusterPhyId, connector.getV2() ),
3000, 3000,
() -> { () -> {
String dsl = dslLoaderUtil.getFormatDslByFileName( String dsl = dslLoaderUtil.getFormatDslByFileName(
@@ -152,25 +161,25 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
connector.getV1().toString(), connector.getV1().toString(),
realIndex, realIndex,
dsl, dsl,
s -> handleListESQueryResponse(s, metrics, aggType), s -> handleListESQueryResponse(s, metricNameList, aggType),
3 3
); );
synchronized (table){ synchronized (table) {
for(String metric : metricMap.keySet()){ for(Map.Entry<String/*metricName*/, List<MetricPointVO>> entry: metricMap.entrySet()){
table.put(metric, connector, metricMap.get(metric)); table.put(entry.getKey(), connector, entry.getValue());
} }
} }
}); });
} catch (Exception e) { } catch (Exception e) {
LOGGER.error( LOGGER.error(
"method=listConnectorMetricsByConnectors||clusterPhyId={}||connectorName{}||errMsg=exception!", "method=listMetricsByConnectors||clusterPhyId={}||connectClusterId={}||connectorName{}||errMsg=exception!",
clusterPhyId, connector.getV2(), e clusterPhyId, connector.getV1(), connector.getV2(), e
); );
} }
} }
queryFuture.waitExecute(); esTPService.waitExecute();
return table; return table;
} }
@@ -181,7 +190,7 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
String aggType, String aggType,
int topN, int topN,
Long startTime, Long startTime,
Long endTime){ Long endTime) {
//1、获取需要查下的索引 //1、获取需要查下的索引
String realIndex = realIndex(startTime, endTime); String realIndex = realIndex(startTime, endTime);
@@ -209,46 +218,19 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
); );
} }
/**************************************************** private method ****************************************************/ /**************************************************** private method ****************************************************/
private Table<String/*topic*/, String/*metric*/, MetricPointVO> handleSingleESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){
Table<String, String, MetricPointVO> table = HashBasedTable.create();
Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap){return table;}
for(String metric : metrics){
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> {
try {
if (null != esBucket.getUnusedMap().get(KEY)) {
String topic = esBucket.getUnusedMap().get(KEY).toString();
String value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE).toString();
MetricPointVO metricPoint = new MetricPointVO();
metricPoint.setAggType(aggType);
metricPoint.setValue(value);
metricPoint.setName(metric);
table.put(topic, metric, metricPoint);
}else {
LOGGER.debug("method=handleListESQueryResponse||metric={}||errMsg=get topic is null!", metric);
}
}catch (Exception e){
LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e);
}
});
}
return table;
}
private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){ private Map<String, List<MetricPointVO>> handleListESQueryResponse(ESQueryResponse response, List<String> metrics, String aggType){
Map<String, List<MetricPointVO>> metricMap = new HashMap<>(); Map<String, List<MetricPointVO>> metricMap = new HashMap<>();
Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response); Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap){return metricMap;} if(null == esAggrMap) {
return metricMap;
}
for(String metric : metrics){ for(String metric : metrics) {
List<MetricPointVO> metricPoints = new ArrayList<>(); List<MetricPointVO> metricPoints = new ArrayList<>();
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach( esBucket -> {
@@ -256,17 +238,11 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null != esBucket.getUnusedMap().get(KEY)) {
Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString());
Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE); Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE);
if(value == null){return;} if(value == null){
return;
}
MetricPointVO metricPoint = new MetricPointVO(); metricPoints.add(new MetricPointVO(metric, timestamp, value.toString(), aggType));
metricPoint.setAggType(aggType);
metricPoint.setTimeStamp(timestamp);
metricPoint.setValue(value.toString());
metricPoint.setName(metric);
metricPoints.add(metricPoint);
}else {
LOGGER.info("");
} }
}catch (Exception e){ }catch (Exception e){
LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e); LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e);
@@ -279,10 +255,12 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
return metricMap; return metricMap;
} }
private Map<String, List<Tuple<Long, String>>> handleTopConnectorESQueryResponse(ESQueryResponse response, List<String> metricNameList, int topN){ private Map<String, List<Tuple<Long, String>>> handleTopConnectorESQueryResponse(ESQueryResponse response,
List<String> metricNameList,
int topN) {
Map<String, List<Tuple<Long, String>>> ret = new HashMap<>(); Map<String, List<Tuple<Long, String>>> ret = new HashMap<>();
Map<String, ESAggr> esAggrMap = checkBucketsAndHitsOfResponseAggs(response); Map<String, ESAggr> esAggrMap = this.checkBucketsAndHitsOfResponseAggs(response);
if(null == esAggrMap) { if(null == esAggrMap) {
return ret; return ret;
} }
@@ -291,7 +269,7 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
// 1、先获取每个指标对应的所有 connector 以及指标的值 // 1、先获取每个指标对应的所有 connector 以及指标的值
for(String metricName : metricNameList) { for(String metricName : metricNameList) {
esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { esAggrMap.get(HIST).getBucketList().forEach(esBucket -> {
try { try {
if (null != esBucket.getUnusedMap().get(KEY)) { if (null != esBucket.getUnusedMap().get(KEY)) {
String connectorNameAndClusterId = esBucket.getUnusedMap().get(KEY).toString(); String connectorNameAndClusterId = esBucket.getUnusedMap().get(KEY).toString();
@@ -338,24 +316,6 @@ public class ConnectorMetricESDAO extends BaseMetricESDAO {
return ret; return ret;
} }
private Map<String/*metric*/, Map<String/*topic*/, List<MetricPointVO>>> topicMetricMap2MetricTopicMap(
Map<String/*topic*/, Map<String/*metric*/, List<MetricPointVO>>> topicMetricMap){
Map<String/*metric*/, Map<String/*topic*/, List<MetricPointVO>>> ret = new HashMap<>();
for(String topic : topicMetricMap.keySet()){
Map<String/*metric*/, List<MetricPointVO>> metricMap = topicMetricMap.get(topic);
for(String metric : metricMap.keySet()){
Map<String/*topic*/, List<MetricPointVO>> brokerMap = (null == ret.get(metric)) ? new HashMap<>() : ret.get(metric);
brokerMap.put(topic, metricMap.get(metric));
ret.put(metric, brokerMap);
}
}
return ret;
}
private Tuple<String, Long> splitConnectorNameAndClusterId(String connectorNameAndClusterId){ private Tuple<String, Long> splitConnectorNameAndClusterId(String connectorNameAndClusterId){
String[] ss = connectorNameAndClusterId.split("#"); String[] ss = connectorNameAndClusterId.split("#");
if(null == ss || ss.length != 2){return null;} if(null == ss || ss.length != 2){return null;}

6
km-rest/src/main/resources/application.yml
View File

@@ -50,7 +50,6 @@ logging:
thread-pool: thread-pool:
scheduled: scheduled:
thread-num: 2 # @Scheduled任务的线程池大小，默认是一个 thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
collector: # 采集模块的配置 collector: # 采集模块的配置
future-util: # 采集模块线程池配置 future-util: # 采集模块线程池配置
num: 3 # 线程池个数 num: 3 # 线程池个数
@@ -58,7 +57,6 @@ thread-pool:
queue-size: 10000 # 每个线程池队列大小 queue-size: 10000 # 每个线程池队列大小
select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改 select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改
suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改 suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改
task: # 任务模块的配置 task: # 任务模块的配置
metrics: # metrics采集任务配置 metrics: # metrics采集任务配置
thread-num: 18 # metrics采集任务线程池核心线程数 thread-num: 18 # metrics采集任务线程池核心线程数
@@ -69,6 +67,10 @@ thread-pool:
common: # 剩余其他任务配置 common: # 剩余其他任务配置
thread-num: 15 # 剩余其他任务线程池核心线程数 thread-num: 15 # 剩余其他任务线程池核心线程数
queue-size: 150 # 剩余其他任务线程池队列大小 queue-size: 150 # 剩余其他任务线程池队列大小
es:
search: # es查询线程池
thread-num: 10 # 线程池大小
queue-size: 5000 # 队列大小
# 客户端池大小相关配置 # 客户端池大小相关配置