From 249fe7c7001df9fa8dc5e12eced4f0fcc6e820b7 Mon Sep 17 00:00:00 2001 From: zengqiao Date: Tue, 6 Dec 2022 17:59:39 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4ES=E7=9B=B8=E5=85=B3=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=BD=8D=E7=BD=AE=20&=20=E8=A1=A5=E5=85=85connectESDA?= =?UTF-8?q?O=E7=9B=B8=E5=85=B3=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connect/ConnectClusterMetrics.java | 35 + .../metrics/connect/ConnectWorkerMetrics.java | 35 + .../metrics/connect/ConnectorMetrics.java | 39 + .../metrics/connect/ConnectorTaskMetrics.java | 39 + .../connect/ConnectClusterMetricPO.java | 30 + .../po/metrice/connect/ConnectorMetricPO.java | 39 + .../km/common/constant/ESIndexConstant.java | 709 ------------------ km-dist/init/template/ks_kafka_broker_metric | 3 +- km-dist/init/template/ks_kafka_cluster_metric | 5 +- .../template/ks_kafka_connect_cluster_metric | 86 +++ .../ks_kafka_connect_connector_metric | 194 +++++ km-dist/init/template/ks_kafka_group_metric | 1 - .../init/template/ks_kafka_partition_metric | 1 - .../init/template/ks_kafka_replication_metric | 3 +- km-dist/init/template/ks_kafka_topic_metric | 1 - .../init/template/ks_kafka_zookeeper_metric | 1 - .../km/persistence/es/ESFileLoader.java | 155 ++++ .../persistence/es/dao/BaseMetricESDAO.java | 22 +- .../persistence/es/dao/BrokerMetricESDAO.java | 21 +- .../es/dao/ClusterMetricESDAO.java | 17 +- .../persistence/es/dao/GroupMetricESDAO.java | 21 +- .../es/dao/PartitionMetricESDAO.java | 13 +- .../es/dao/ReplicationMetricESDAO.java | 13 +- .../persistence/es/dao/TopicMetricESDAO.java | 29 +- .../es/dao/ZookeeperMetricESDAO.java | 12 +- .../connect/ConnectClusterMetricESDAO.java | 275 +++++++ .../es/dao/connect/ConnectorMetricESDAO.java | 365 +++++++++ .../{DslsConstant.java => DslConstant.java} | 18 +- .../km/persistence/es/dsls/DslLoaderUtil.java | 183 +---- .../es/template/TemplateConstant.java | 19 + .../es/template/TemplateLoaderUtil.java | 28 + .../dsl/BaseMetricESDAO/getLatestMetricTime | 0 .../BrokerMetricESDAO/getAggListBrokerMetrics | 0 .../getAggSingleBrokerMetrics | 0 .../BrokerMetricESDAO/getAggTopMetricsBrokers | 0 .../BrokerMetricESDAO/getBrokerLatestMetrics | 0 .../getAggListClusterMetrics | 0 .../getAggSingleClusterMetrics | 0 .../getClusterLatestMetrics | 0 .../listClusterWithLatestMetrics | 0 .../getAggListConnectClusterMetrics | 44 ++ .../getAggTopMetricsConnectClusters | 45 ++ .../getConnectorAggListMetric | 51 ++ .../getConnectorAggTopMetric | 45 ++ .../getConnectorLatestMetric | 39 + .../GroupMetricESDAO/countGroupMetricValue | 0 .../GroupMetricESDAO/countGroupNotMetricValue | 0 .../GroupMetricESDAO/getTopicPartitionOfGroup | 0 .../dsl/GroupMetricESDAO/listGroupMetrics | 0 .../listLatestMetricsAggByGroupTopic | 0 .../listPartitionLatestMetrics | 0 .../getPartitionLatestMetrics | 0 .../listPartitionLatestMetricsByTopic | 0 .../getAggSingleReplicationMetrics | 0 .../getReplicationLatestMetrics | 0 .../TopicMetricESDAO/countTopicMetricValue | 0 .../TopicMetricESDAO/countTopicNotMetricValue | 0 .../dsl/TopicMetricESDAO/getAggListMetrics | 0 .../dsl/TopicMetricESDAO/getAggSingleMetrics | 0 .../TopicMetricESDAO/getAggTopMetricsTopics | 0 .../TopicMetricESDAO/getMaxOrMinSingleMetric | 0 .../dsl/TopicMetricESDAO/getTopicLatestMetric | 0 .../getTopicLatestMetricByBrokerId | 0 .../listTopicWithLatestMetrics | 0 .../getAggListZookeeperMetrics | 0 .../es/template/ks_kafka_broker_metric | 101 +++ .../es/template/ks_kafka_cluster_metric | 186 +++++ .../template/ks_kafka_connect_cluster_metric | 86 +++ .../ks_kafka_connect_connector_metric | 194 +++++ .../es/template/ks_kafka_group_metric | 74 ++ .../es/template/ks_kafka_partition_metric | 65 ++ .../es/template/ks_kafka_replication_metric | 65 ++ .../es/template/ks_kafka_topic_metric | 116 +++ .../es/template/ks_kafka_zookeeper_metric | 84 +++ 74 files changed, 2640 insertions(+), 967 deletions(-) create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectClusterMetrics.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectWorkerMetrics.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorMetrics.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorTaskMetrics.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectClusterMetricPO.java create mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectorMetricPO.java delete mode 100644 km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/ESIndexConstant.java create mode 100644 km-dist/init/template/ks_kafka_connect_cluster_metric create mode 100644 km-dist/init/template/ks_kafka_connect_connector_metric create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/ESFileLoader.java create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectClusterMetricESDAO.java create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectorMetricESDAO.java rename km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/{DslsConstant.java => DslConstant.java} (83%) create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateConstant.java create mode 100644 km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateLoaderUtil.java rename km-persistence/src/main/resources/{ => es}/dsl/BaseMetricESDAO/getLatestMetricTime (100%) rename km-persistence/src/main/resources/{ => es}/dsl/BrokerMetricESDAO/getAggListBrokerMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/BrokerMetricESDAO/getAggSingleBrokerMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/BrokerMetricESDAO/getAggTopMetricsBrokers (100%) rename km-persistence/src/main/resources/{ => es}/dsl/BrokerMetricESDAO/getBrokerLatestMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ClusterMetricESDAO/getAggListClusterMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ClusterMetricESDAO/getAggSingleClusterMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ClusterMetricESDAO/getClusterLatestMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ClusterMetricESDAO/listClusterWithLatestMetrics (100%) create mode 100644 km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggListConnectClusterMetrics create mode 100644 km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggTopMetricsConnectClusters create mode 100644 km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggListMetric create mode 100644 km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggTopMetric create mode 100644 km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorLatestMetric rename km-persistence/src/main/resources/{ => es}/dsl/GroupMetricESDAO/countGroupMetricValue (100%) rename km-persistence/src/main/resources/{ => es}/dsl/GroupMetricESDAO/countGroupNotMetricValue (100%) rename km-persistence/src/main/resources/{ => es}/dsl/GroupMetricESDAO/getTopicPartitionOfGroup (100%) rename km-persistence/src/main/resources/{ => es}/dsl/GroupMetricESDAO/listGroupMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/GroupMetricESDAO/listLatestMetricsAggByGroupTopic (100%) rename km-persistence/src/main/resources/{ => es}/dsl/GroupMetricESDAO/listPartitionLatestMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/PartitionMetricESDAO/getPartitionLatestMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/PartitionMetricESDAO/listPartitionLatestMetricsByTopic (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ReplicationMetricESDAO/getAggSingleReplicationMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ReplicationMetricESDAO/getReplicationLatestMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/countTopicMetricValue (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/countTopicNotMetricValue (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/getAggListMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/getAggSingleMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/getAggTopMetricsTopics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/getMaxOrMinSingleMetric (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/getTopicLatestMetric (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/getTopicLatestMetricByBrokerId (100%) rename km-persistence/src/main/resources/{ => es}/dsl/TopicMetricESDAO/listTopicWithLatestMetrics (100%) rename km-persistence/src/main/resources/{ => es}/dsl/ZookeeperMetricESDAO/getAggListZookeeperMetrics (100%) create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_broker_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_cluster_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_connect_cluster_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_connect_connector_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_group_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_partition_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_replication_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_topic_metric create mode 100644 km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectClusterMetrics.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectClusterMetrics.java new file mode 100644 index 00000000..fe710391 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectClusterMetrics.java @@ -0,0 +1,35 @@ +package com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect; + +import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +/** + * @author zengqiao + * @date 20/6/17 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@ToString +public class ConnectClusterMetrics extends BaseMetrics { + private Long connectClusterId; + + public ConnectClusterMetrics(Long clusterPhyId, Long connectClusterId){ + super(clusterPhyId); + this.connectClusterId = connectClusterId; + } + + public static ConnectClusterMetrics initWithMetric(Long connectClusterId, String metric, Float value) { + ConnectClusterMetrics brokerMetrics = new ConnectClusterMetrics(connectClusterId, connectClusterId); + brokerMetrics.putMetric(metric, value); + return brokerMetrics; + } + + @Override + public String unique() { + return "KCC@" + clusterPhyId + "@" + connectClusterId; + } +} \ No newline at end of file diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectWorkerMetrics.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectWorkerMetrics.java new file mode 100644 index 00000000..78d9fe06 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectWorkerMetrics.java @@ -0,0 +1,35 @@ +package com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect; + +import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +/** + * @author wyb + * @date 2022/11/2 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@ToString +public class ConnectWorkerMetrics extends BaseMetrics { + + private Long connectClusterId; + + private String workerId; + + public static ConnectWorkerMetrics initWithMetric(Long connectClusterId, String workerId, String metric, Float value) { + ConnectWorkerMetrics connectWorkerMetrics = new ConnectWorkerMetrics(); + connectWorkerMetrics.setConnectClusterId(connectClusterId); + connectWorkerMetrics.setWorkerId(workerId); + connectWorkerMetrics.putMetric(metric, value); + return connectWorkerMetrics; + } + + @Override + public String unique() { + return "KCC@" + clusterPhyId + "@" + connectClusterId + "@" + workerId; + } +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorMetrics.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorMetrics.java new file mode 100644 index 00000000..08540ed5 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorMetrics.java @@ -0,0 +1,39 @@ +package com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect; + +import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +/** + * @author zengqiao + * @date 20/6/17 + */ +@Data +@NoArgsConstructor +@ToString +public class ConnectorMetrics extends BaseMetrics { + private Long connectClusterId; + + private String connectorName; + + private String connectorNameAndClusterId; + + public ConnectorMetrics(Long connectClusterId, String connectorName) { + super(null); + this.connectClusterId = connectClusterId; + this.connectorName = connectorName; + this.connectorNameAndClusterId = connectorName + "#" + connectClusterId; + } + + public static ConnectorMetrics initWithMetric(Long connectClusterId, String connectorName, String metricName, Float value) { + ConnectorMetrics metrics = new ConnectorMetrics(connectClusterId, connectorName); + metrics.putMetric(metricName, value); + return metrics; + } + + @Override + public String unique() { + return "KCOR@" + connectClusterId + "@" + connectorName; + } +} \ No newline at end of file diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorTaskMetrics.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorTaskMetrics.java new file mode 100644 index 00000000..26b244e7 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/entity/metrics/connect/ConnectorTaskMetrics.java @@ -0,0 +1,39 @@ +package com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.connect; + +import com.xiaojukeji.know.streaming.km.common.bean.entity.metrics.BaseMetrics; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +/** + * @author wyb + * @date 2022/11/4 + */ +@Data +@NoArgsConstructor +@ToString +public class ConnectorTaskMetrics extends BaseMetrics { + private Long connectClusterId; + + private String connectorName; + + private Integer taskId; + + public ConnectorTaskMetrics(Long connectClusterId, String connectorName, Integer taskId) { + this.connectClusterId = connectClusterId; + this.connectorName = connectorName; + this.taskId = taskId; + } + + public static ConnectorTaskMetrics initWithMetric(Long connectClusterId, String connectorName, Integer taskId, String metricName, Float value) { + ConnectorTaskMetrics metrics = new ConnectorTaskMetrics(connectClusterId, connectorName, taskId); + metrics.putMetric(metricName,value); + return metrics; + } + + @Override + public String unique() { + return "KCOR@" + connectClusterId + "@" + connectorName + "@" + taskId; + } +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectClusterMetricPO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectClusterMetricPO.java new file mode 100644 index 00000000..462c7763 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectClusterMetricPO.java @@ -0,0 +1,30 @@ +package com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect; + +import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.BaseMetricESPO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import static com.xiaojukeji.know.streaming.km.common.utils.CommonUtils.monitorTimestamp2min; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ConnectClusterMetricPO extends BaseMetricESPO { + private Long connectClusterId; + + public ConnectClusterMetricPO(Long kafkaClusterPhyId, Long connectClusterId){ + super(kafkaClusterPhyId); + this.connectClusterId = connectClusterId; + } + + @Override + public String getKey() { + return "KCC@" + clusterPhyId + "@" + connectClusterId + "@" + monitorTimestamp2min(timestamp); + } + + @Override + public String getRoutingValue() { + return String.valueOf(connectClusterId); + } +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectorMetricPO.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectorMetricPO.java new file mode 100644 index 00000000..7f3f7d71 --- /dev/null +++ b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/bean/po/metrice/connect/ConnectorMetricPO.java @@ -0,0 +1,39 @@ +package com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect; + +import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.BaseMetricESPO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import static com.xiaojukeji.know.streaming.km.common.utils.CommonUtils.monitorTimestamp2min; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ConnectorMetricPO extends BaseMetricESPO { + private Long connectClusterId; + + private String connectorName; + + /** + * 用于es内部排序 + */ + private String connectorNameAndClusterId; + + public ConnectorMetricPO(Long kafkaClusterPhyId, Long connectClusterId, String connectorName){ + super(kafkaClusterPhyId); + this.connectClusterId = connectClusterId; + this.connectorName = connectorName; + this.connectorNameAndClusterId = connectorName + "#" + connectClusterId; + } + + @Override + public String getKey() { + return "KCOR@" + clusterPhyId + "@" + connectClusterId + "@" + connectorName + "@" + monitorTimestamp2min(timestamp); + } + + @Override + public String getRoutingValue() { + return String.valueOf(connectClusterId); + } +} diff --git a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/ESIndexConstant.java b/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/ESIndexConstant.java deleted file mode 100644 index 1f5654d0..00000000 --- a/km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/ESIndexConstant.java +++ /dev/null @@ -1,709 +0,0 @@ -package com.xiaojukeji.know.streaming.km.common.constant; - -public class ESIndexConstant { - - public final static String TOPIC_INDEX = "ks_kafka_topic_metric"; - public final static String TOPIC_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_topic_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"brokerId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"topic\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"BytesIn_min_15\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"Messages\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesRejected\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"PartitionURP\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthCheckTotal\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"ReplicationCount\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"ReplicationBytesOut\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"ReplicationBytesIn\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"FailedFetchRequests\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesIn_min_5\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthScore\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"LogSize\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesOut\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesOut_min_15\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"FailedProduceRequests\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesIn\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesOut_min_5\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"MessagesIn\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"TotalProduceRequests\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthCheckPassed\" : {\n" + - " \"type\" : \"float\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"brokerAgg\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"index\" : true,\n" + - " \"type\" : \"date\",\n" + - " \"doc_values\" : true\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; - - public final static String CLUSTER_INDEX = "ks_kafka_cluster_metric"; - public final static String CLUSTER_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_cluster_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"Connections\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"BytesIn_min_15\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"PartitionURP\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthScore_Topics\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"EventQueueSize\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"ActiveControllerCount\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"GroupDeads\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"BytesIn_min_5\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckTotal_Topics\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Partitions\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"BytesOut\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Groups\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"BytesOut_min_15\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"TotalRequestQueueSize\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckPassed_Groups\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"TotalProduceRequests\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckPassed\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"TotalLogSize\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"GroupEmptys\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"PartitionNoLeader\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthScore_Brokers\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Messages\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Topics\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"PartitionMinISR_E\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckTotal\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Brokers\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Replicas\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckTotal_Groups\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"GroupRebalances\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"MessageIn\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthScore\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckPassed_Topics\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckTotal_Brokers\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"PartitionMinISR_S\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"BytesIn\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"BytesOut_min_5\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"GroupActives\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"MessagesIn\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"GroupReBalances\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckPassed_Brokers\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthScore_Groups\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"TotalResponseQueueSize\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"Zookeepers\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"LeaderMessages\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthScore_Cluster\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckPassed_Cluster\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"HealthCheckTotal_Cluster\" : {\n" + - " \"type\" : \"double\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"type\" : \"date\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; - - public final static String BROKER_INDEX = "ks_kafka_broker_metric"; - public final static String BROKER_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_broker_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"brokerId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"NetworkProcessorAvgIdle\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"UnderReplicatedPartitions\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesIn_min_15\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthCheckTotal\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"RequestHandlerAvgIdle\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"connectionsCount\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesIn_min_5\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthScore\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesOut\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesOut_min_15\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesIn\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"BytesOut_min_5\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"TotalRequestQueueSize\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"MessagesIn\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"TotalProduceRequests\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthCheckPassed\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"TotalResponseQueueSize\" : {\n" + - " \"type\" : \"float\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"index\" : true,\n" + - " \"type\" : \"date\",\n" + - " \"doc_values\" : true\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; - - public final static String PARTITION_INDEX = "ks_kafka_partition_metric"; - public final static String PARTITION_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_partition_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"brokerId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"partitionId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"topic\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"LogStartOffset\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"Messages\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"LogEndOffset\" : {\n" + - " \"type\" : \"float\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"index\" : true,\n" + - " \"type\" : \"date\",\n" + - " \"doc_values\" : true\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; - - public final static String GROUP_INDEX = "ks_kafka_group_metric"; - public final static String GROUP_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_group_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"group\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"partitionId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"topic\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"HealthScore\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"Lag\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"OffsetConsumed\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthCheckTotal\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"HealthCheckPassed\" : {\n" + - " \"type\" : \"float\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"groupMetric\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"index\" : true,\n" + - " \"type\" : \"date\",\n" + - " \"doc_values\" : true\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; - - public final static String REPLICATION_INDEX = "ks_kafka_replication_metric"; - public final static String REPLICATION_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_replication_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"brokerId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"partitionId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"topic\" : {\n" + - " \"type\" : \"keyword\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"LogStartOffset\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"Messages\" : {\n" + - " \"type\" : \"float\"\n" + - " },\n" + - " \"LogEndOffset\" : {\n" + - " \"type\" : \"float\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"index\" : true,\n" + - " \"type\" : \"date\",\n" + - " \"doc_values\" : true\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; - - public final static String ZOOKEEPER_INDEX = "ks_kafka_zookeeper_metric"; - public final static String ZOOKEEPER_TEMPLATE = "{\n" + - " \"order\" : 10,\n" + - " \"index_patterns\" : [\n" + - " \"ks_kafka_zookeeper_metric*\"\n" + - " ],\n" + - " \"settings\" : {\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : \"10\"\n" + - " }\n" + - " },\n" + - " \"mappings\" : {\n" + - " \"properties\" : {\n" + - " \"routingValue\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"clusterPhyId\" : {\n" + - " \"type\" : \"long\"\n" + - " },\n" + - " \"metrics\" : {\n" + - " \"properties\" : {\n" + - " \"AvgRequestLatency\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"MinRequestLatency\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"MaxRequestLatency\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"OutstandingRequests\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"NodeCount\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"WatchCount\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"NumAliveConnections\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"PacketsReceived\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"PacketsSent\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"EphemeralsCount\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"ApproximateDataSize\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"OpenFileDescriptorCount\" : {\n" + - " \"type\" : \"double\"\n" + - " },\n" + - " \"MaxFileDescriptorCount\" : {\n" + - " \"type\" : \"double\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"key\" : {\n" + - " \"type\" : \"text\",\n" + - " \"fields\" : {\n" + - " \"keyword\" : {\n" + - " \"ignore_above\" : 256,\n" + - " \"type\" : \"keyword\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"timestamp\" : {\n" + - " \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" + - " \"type\" : \"date\"\n" + - " }\n" + - " }\n" + - " },\n" + - " \"aliases\" : { }\n" + - " }"; -} diff --git a/km-dist/init/template/ks_kafka_broker_metric b/km-dist/init/template/ks_kafka_broker_metric index 78933a4d..749b1494 100644 --- a/km-dist/init/template/ks_kafka_broker_metric +++ b/km-dist/init/template/ks_kafka_broker_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_broker_metric { "order" : 10, "index_patterns" : [ @@ -6,7 +5,7 @@ PUT _template/ks_kafka_broker_metric ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : { diff --git a/km-dist/init/template/ks_kafka_cluster_metric b/km-dist/init/template/ks_kafka_cluster_metric index 76d51cbd..942fdce2 100644 --- a/km-dist/init/template/ks_kafka_cluster_metric +++ b/km-dist/init/template/ks_kafka_cluster_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_cluster_metric { "order" : 10, "index_patterns" : [ @@ -6,7 +5,7 @@ PUT _template/ks_kafka_cluster_metric ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : { @@ -184,4 +183,4 @@ PUT _template/ks_kafka_cluster_metric } }, "aliases" : { } - } \ No newline at end of file + } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_connect_cluster_metric b/km-dist/init/template/ks_kafka_connect_cluster_metric new file mode 100644 index 00000000..7fa4c523 --- /dev/null +++ b/km-dist/init/template/ks_kafka_connect_cluster_metric @@ -0,0 +1,86 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_connect_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "connectClusterId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "ConnectorCount" : { + "type" : "float" + }, + "TaskCount" : { + "type" : "float" + }, + "ConnectorStartupAttemptsTotal" : { + "type" : "float" + }, + "ConnectorStartupFailurePercentage" : { + "type" : "float" + }, + "ConnectorStartupFailureTotal" : { + "type" : "float" + }, + "ConnectorStartupSuccessPercentage" : { + "type" : "float" + }, + "ConnectorStartupSuccessTotal" : { + "type" : "float" + }, + "TaskStartupAttemptsTotal" : { + "type" : "float" + }, + "TaskStartupFailurePercentage" : { + "type" : "float" + }, + "TaskStartupFailureTotal" : { + "type" : "float" + }, + "TaskStartupSuccessPercentage" : { + "type" : "float" + }, + "TaskStartupSuccessTotal" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_connect_connector_metric b/km-dist/init/template/ks_kafka_connect_connector_metric new file mode 100644 index 00000000..b26836a0 --- /dev/null +++ b/km-dist/init/template/ks_kafka_connect_connector_metric @@ -0,0 +1,194 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_connect_connector_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "connectClusterId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "connectorName" : { + "type" : "keyword" + }, + "connectorNameAndClusterId" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "HealthState" : { + "type" : "float" + }, + "ConnectorTotalTaskCount" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ConnectorRunningTaskCount" : { + "type" : "float" + }, + "ConnectorPausedTaskCount" : { + "type" : "float" + }, + "ConnectorFailedTaskCount" : { + "type" : "float" + }, + "ConnectorUnassignedTaskCount" : { + "type" : "float" + }, + "BatchSizeAvg" : { + "type" : "float" + }, + "BatchSizeMax" : { + "type" : "float" + }, + "OffsetCommitAvgTimeMs" : { + "type" : "float" + }, + "OffsetCommitMaxTimeMs" : { + "type" : "float" + }, + "OffsetCommitFailurePercentage" : { + "type" : "float" + }, + "OffsetCommitSuccessPercentage" : { + "type" : "float" + }, + "PollBatchAvgTimeMs" : { + "type" : "float" + }, + "PollBatchMaxTimeMs" : { + "type" : "float" + }, + "SourceRecordActiveCount" : { + "type" : "float" + }, + "SourceRecordActiveCountAvg" : { + "type" : "float" + }, + "SourceRecordActiveCountMax" : { + "type" : "float" + }, + "SourceRecordPollRate" : { + "type" : "float" + }, + "SourceRecordPollTotal" : { + "type" : "float" + }, + "SourceRecordWriteRate" : { + "type" : "float" + }, + "SourceRecordWriteTotal" : { + "type" : "float" + }, + "OffsetCommitCompletionRate" : { + "type" : "float" + }, + "OffsetCommitCompletionTotal" : { + "type" : "float" + }, + "OffsetCommitSkipRate" : { + "type" : "float" + }, + "OffsetCommitSkipTotal" : { + "type" : "float" + }, + "PartitionCount" : { + "type" : "float" + }, + "PutBatchAvgTimeMs" : { + "type" : "float" + }, + "PutBatchMaxTimeMs" : { + "type" : "float" + }, + "SinkRecordActiveCount" : { + "type" : "float" + }, + "SinkRecordActiveCountAvg" : { + "type" : "float" + }, + "SinkRecordActiveCountMax" : { + "type" : "float" + }, + "SinkRecordLagMax" : { + "type" : "float" + }, + "SinkRecordReadRate" : { + "type" : "float" + }, + "SinkRecordReadTotal" : { + "type" : "float" + }, + "SinkRecordSendRate" : { + "type" : "float" + }, + "SinkRecordSendTotal" : { + "type" : "float" + }, + "DeadletterqueueProduceFailures" : { + "type" : "float" + }, + "DeadletterqueueProduceRequests" : { + "type" : "float" + }, + "LastErrorTimestamp" : { + "type" : "float" + }, + "TotalErrorsLogged" : { + "type" : "float" + }, + "TotalRecordErrors" : { + "type" : "float" + }, + "TotalRecordFailures" : { + "type" : "float" + }, + "TotalRecordsSkipped" : { + "type" : "float" + }, + "TotalRetries" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_group_metric b/km-dist/init/template/ks_kafka_group_metric index 9fe7008b..3f04f16a 100644 --- a/km-dist/init/template/ks_kafka_group_metric +++ b/km-dist/init/template/ks_kafka_group_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_group_metric { "order" : 10, "index_patterns" : [ diff --git a/km-dist/init/template/ks_kafka_partition_metric b/km-dist/init/template/ks_kafka_partition_metric index 2538eba3..68264660 100644 --- a/km-dist/init/template/ks_kafka_partition_metric +++ b/km-dist/init/template/ks_kafka_partition_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_partition_metric { "order" : 10, "index_patterns" : [ diff --git a/km-dist/init/template/ks_kafka_replication_metric b/km-dist/init/template/ks_kafka_replication_metric index c9c254be..a8ff4b53 100644 --- a/km-dist/init/template/ks_kafka_replication_metric +++ b/km-dist/init/template/ks_kafka_replication_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_replication_metric { "order" : 10, "index_patterns" : [ @@ -63,4 +62,4 @@ PUT _template/ks_kafka_replication_metric } }, "aliases" : { } - } + } \ No newline at end of file diff --git a/km-dist/init/template/ks_kafka_topic_metric b/km-dist/init/template/ks_kafka_topic_metric index a2456dc6..b2eed35d 100644 --- a/km-dist/init/template/ks_kafka_topic_metric +++ b/km-dist/init/template/ks_kafka_topic_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_topic_metric { "order" : 10, "index_patterns" : [ diff --git a/km-dist/init/template/ks_kafka_zookeeper_metric b/km-dist/init/template/ks_kafka_zookeeper_metric index abb54a61..e2efb41a 100644 --- a/km-dist/init/template/ks_kafka_zookeeper_metric +++ b/km-dist/init/template/ks_kafka_zookeeper_metric @@ -1,4 +1,3 @@ -PUT _template/ks_kafka_zookeeper_metric { "order" : 10, "index_patterns" : [ diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/ESFileLoader.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/ESFileLoader.java new file mode 100644 index 00000000..2702f329 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/ESFileLoader.java @@ -0,0 +1,155 @@ +package com.xiaojukeji.know.streaming.km.persistence.es; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.parser.DefaultJSONParser; +import com.alibaba.fastjson.parser.Feature; +import com.alibaba.fastjson.parser.ParserConfig; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.didiglobal.logi.log.ILog; +import com.google.common.collect.Lists; +import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author didi + */ +public class ESFileLoader { + private static final ILog LOGGER = LoggerUtil.getESLogger(); + + public Map loaderFileContext(String filePath, Field[] fields) { + LOGGER.info("method=loaderFileContext||DslLoaderUtil init start."); + List dslFileNames = Lists.newLinkedList(); + Map fileContextMap = new HashMap<>(); + + if(null == fields || 0 == fields.length){ + return fileContextMap; + } + + // 反射获取接口中定义的变量中的值 + for (int i = 0; i < fields.length; ++i) { + fields[i].setAccessible(true); + try { + dslFileNames.add(fields[i].get(null).toString()); + } catch (IllegalAccessException e) { + LOGGER.error("method=loaderFileContext||errMsg=fail to read {} error. ", fields[i].getName(), + e); + } + } + + // 加载dsl文件及内容 + for (String fileName : dslFileNames) { + fileContextMap.put(fileName, readEsFileInJarFile(filePath, fileName)); + } + + // 输出加载的查询语句 + LOGGER.info("method=loaderFileContext||msg=dsl files count {}", fileContextMap.size()); + for (Map.Entry entry : fileContextMap.entrySet()) { + LOGGER.info("method=loaderFileContext||msg=file name {}, dsl content {}", entry.getKey(), + entry.getValue()); + } + + LOGGER.info("method=loaderFileContext||DslLoaderUtil init finished."); + + return fileContextMap; + } + + /** + * 去除json中的空格 + * + * @param sourceDsl + * @return + */ + public String trimJsonBank(String sourceDsl) { + List dslList = Lists.newArrayList(); + + DefaultJSONParser parser = null; + Object obj = null; + String dsl = sourceDsl; + + // 解析多个json，直到pos为0 + for (;;) { + try { + // 这里需要Feature.OrderedField.getMask()保持有序 + parser = new DefaultJSONParser(dsl, ParserConfig.getGlobalInstance(), + JSON.DEFAULT_PARSER_FEATURE | Feature.OrderedField.getMask()); + obj = parser.parse(); + } catch (Exception t) { + LOGGER.error("method=trimJsonBank||errMsg=parse json {} error. ", dsl, t); + } + if (obj == null) { + break; + } + + if (obj instanceof JSONObject) { + dslList.add( JSON.toJSONString(obj, SerializerFeature.WriteMapNullValue)); + int pos = parser.getLexer().pos(); + if (pos <= 0) { + break; + } + dsl = dsl.substring(pos); + parser.getLexer().close(); + } else { + parser.getLexer().close(); + break; + } + } + + // 格式化异常或者有多个查询语句，返回原来的查询语句 + if (dslList.isEmpty() || dslList.size() > 1) { + return sourceDsl; + } + + return dslList.get(0); + } + + /** + * 从jar包中读取es相关的语句文件 + * + * @param fileName + * @return + */ + private String readEsFileInJarFile(String filePath, String fileName) { + InputStream inputStream = this.getClass().getClassLoader() + .getResourceAsStream( filePath + fileName); + + if (inputStream != null) { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + String line = null; + List lines = Lists.newLinkedList(); + try { + while ((line = bufferedReader.readLine()) != null) { + lines.add(line); + } + return StringUtils.join(lines, ""); + + } catch (IOException e) { + LOGGER.error("method=readDslFileInJarFile||errMsg=read file {} error. ", fileName, + e); + + return ""; + } finally { + try { + inputStream.close(); + } catch (IOException e) { + LOGGER.error( + "method=readDslFileInJarFile||errMsg=fail to close file {} error. ", + fileName, e); + } + } + } else { + LOGGER.error("method=readDslFileInJarFile||errMsg=fail to read file {} content", + fileName); + return ""; + } + } +} diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BaseMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BaseMetricESDAO.java index 568ac34e..609b63d0 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BaseMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BaseMetricESDAO.java @@ -13,8 +13,10 @@ import com.xiaojukeji.know.streaming.km.common.utils.CommonUtils; import com.xiaojukeji.know.streaming.km.common.utils.EnvUtil; import com.xiaojukeji.know.streaming.km.common.utils.IndexNameUtils; import com.xiaojukeji.know.streaming.km.persistence.es.BaseESDAO; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateLoaderUtil; import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.util.CollectionUtils; @@ -29,8 +31,7 @@ public class BaseMetricESDAO extends BaseESDAO { /** * 操作的索引名称 */ - protected String indexName; - protected String indexTemplate; + protected String indexName; protected static final Long ONE_MIN = 60 * 1000L; protected static final Long FIVE_MIN = 5 * ONE_MIN; @@ -44,6 +45,9 @@ public class BaseMetricESDAO extends BaseESDAO { */ private static Map ariusStatsEsDaoMap = Maps.newConcurrentMap(); + @Autowired + private TemplateLoaderUtil templateLoaderUtil; + /** * es 地址 */ @@ -56,6 +60,7 @@ public class BaseMetricESDAO extends BaseESDAO { @Scheduled(cron = "0 3/5 * * * ?") public void checkCurrentDayIndexExist(){ try { + String indexTemplate = templateLoaderUtil.getContextByFileName(indexName); esOpClient.createIndexTemplateIfNotExist(indexName, indexTemplate); //检查最近7天索引存在不存 @@ -103,6 +108,15 @@ public class BaseMetricESDAO extends BaseESDAO { ariusStatsEsDaoMap.put(statsType, baseAriusStatsEsDao); } + /** + * 注册不同维度数据对应操作的es类 + * + * @param baseAriusStatsEsDao + */ + public void register(BaseMetricESDAO baseAriusStatsEsDao) { + BaseMetricESDAO.register(indexName, baseAriusStatsEsDao); + } + /** * 批量插入索引统计信息 @@ -416,7 +430,7 @@ public class BaseMetricESDAO extends BaseESDAO { Long endTime = System.currentTimeMillis(); Long startTime = endTime - 12 * ONE_HOUR; - String dsl = dslLoaderUtil.getFormatDslByFileName(DslsConstant.GET_LATEST_METRIC_TIME, startTime, endTime, appendQueryDsl); + String dsl = dslLoaderUtil.getFormatDslByFileName( DslConstant.GET_LATEST_METRIC_TIME, startTime, endTime, appendQueryDsl); String realIndexName = IndexNameUtils.genDailyIndexName(indexName, startTime, endTime); return esOpClient.performRequest( diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BrokerMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BrokerMetricESDAO.java index 7ee76a3e..d3d52f9e 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BrokerMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/BrokerMetricESDAO.java @@ -9,7 +9,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPoint import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.Tuple; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -18,16 +18,15 @@ import java.util.*; import java.util.stream.Collectors; import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.BROKER_INDEX; @Component public class BrokerMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = BROKER_INDEX; - super.indexTemplate = BROKER_TEMPLATE; + super.indexName = BROKER_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register( this); } protected FutureWaitUtil queryFuture = FutureWaitUtil.init("BrokerMetricESDAO", 4,8, 500); @@ -40,7 +39,7 @@ public class BrokerMetricESDAO extends BaseMetricESDAO { Long startTime = endTime - FIVE_MIN; String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_BROKER_LATEST_METRICS, clusterId, brokerId, startTime, endTime); + DslConstant.GET_BROKER_LATEST_METRICS, clusterId, brokerId, startTime, endTime); BrokerMetricPO brokerMetricPO = esOpClient.performRequestAndTakeFirst( brokerId.toString(), @@ -68,7 +67,7 @@ public class BrokerMetricESDAO extends BaseMetricESDAO { String aggDsl = buildAggsDSL(metrics, aggType); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_BROKER_AGG_SINGLE_METRICS, clusterPhyId, brokerId, startTime, endTime, aggDsl); + DslConstant.GET_BROKER_AGG_SINGLE_METRICS, clusterPhyId, brokerId, startTime, endTime, aggDsl); return esOpClient.performRequestWithRouting( String.valueOf(brokerId), @@ -132,7 +131,7 @@ public class BrokerMetricESDAO extends BaseMetricESDAO { for(Long brokerId : brokerIds){ try { String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_BROKER_AGG_LIST_METRICS, + DslConstant.GET_BROKER_AGG_LIST_METRICS, clusterPhyId, brokerId, startTime, @@ -154,8 +153,8 @@ public class BrokerMetricESDAO extends BaseMetricESDAO { ); synchronized (table) { - for(String metric : metricMap.keySet()){ - table.put(metric, brokerId, metricMap.get(metric)); + for(Map.Entry> entry: metricMap.entrySet()){ + table.put(entry.getKey(), brokerId, entry.getValue()); } } }); @@ -187,7 +186,7 @@ public class BrokerMetricESDAO extends BaseMetricESDAO { //4、查询es String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_BROKER_AGG_TOP_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); + DslConstant.GET_BROKER_AGG_TOP_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); return esOpClient.performRequest(realIndex, dsl, s -> handleTopBrokerESQueryResponse(s, metrics, topN), 3); diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ClusterMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ClusterMetricESDAO.java index d53f83bf..5d53aa16 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ClusterMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ClusterMetricESDAO.java @@ -12,7 +12,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ClusterMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @@ -23,17 +23,16 @@ import java.util.List; import java.util.Map; import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CLUSTER_INDEX; @Component public class ClusterMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = CLUSTER_INDEX; - super.indexTemplate = CLUSTER_TEMPLATE; + super.indexName = CLUSTER_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register(this); } protected FutureWaitUtil queryFuture = FutureWaitUtil.init("ClusterMetricESDAO", 4,8, 500); @@ -46,7 +45,7 @@ public class ClusterMetricESDAO extends BaseMetricESDAO { Long startTime = endTime - FIVE_MIN; String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_CLUSTER_LATEST_METRICS, clusterId, startTime, endTime); + DslConstant.GET_CLUSTER_LATEST_METRICS, clusterId, startTime, endTime); ClusterMetricPO clusterMetricPO = esOpClient.performRequestAndTakeFirst( clusterId.toString(), realIndex(startTime, endTime), dsl, ClusterMetricPO.class); @@ -67,7 +66,7 @@ public class ClusterMetricESDAO extends BaseMetricESDAO { String aggDsl = buildAggsDSL(metrics, aggType); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_CLUSTER_AGG_SINGLE_METRICS, clusterPhyId, startTime, endTime, aggDsl); + DslConstant.GET_CLUSTER_AGG_SINGLE_METRICS, clusterPhyId, startTime, endTime, aggDsl); return esOpClient.performRequestWithRouting(String.valueOf(clusterPhyId), realIndex, dsl, s -> handleSingleESQueryResponse(s, metrics, aggType), 3); @@ -103,7 +102,7 @@ public class ClusterMetricESDAO extends BaseMetricESDAO { } String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.LIST_CLUSTER_WITH_LATEST_METRICS, latestMetricTime, appendQueryDsl.toString(), sortDsl); + DslConstant.LIST_CLUSTER_WITH_LATEST_METRICS, latestMetricTime, appendQueryDsl.toString(), sortDsl); return esOpClient.performRequest(realIndex, dsl, ClusterMetricPO.class); } @@ -133,7 +132,7 @@ public class ClusterMetricESDAO extends BaseMetricESDAO { 5000, () -> { String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_CLUSTER_AGG_LIST_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); + DslConstant.GET_CLUSTER_AGG_LIST_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); Map> metricMap = esOpClient.performRequestWithRouting( String.valueOf(clusterPhyId), realIndex, dsl, diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/GroupMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/GroupMetricESDAO.java index 782adc2f..26ebdea1 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/GroupMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/GroupMetricESDAO.java @@ -12,7 +12,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPoint import com.xiaojukeji.know.streaming.km.common.enums.AggTypeEnum; import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -23,17 +23,16 @@ import java.util.stream.Collectors; import static com.xiaojukeji.know.streaming.km.common.constant.Constant.ZERO; import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.GROUP_INDEX; @Component public class GroupMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = GROUP_INDEX; - super.indexTemplate = GROUP_TEMPLATE; + super.indexName = GROUP_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register(this); } protected FutureWaitUtil queryFuture = FutureWaitUtil.init("GroupMetricESDAO", 4,8, 500); @@ -59,7 +58,7 @@ public class GroupMetricESDAO extends BaseMetricESDAO { String topic = groupTopic.getTopicName(); try { String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.LIST_GROUP_LATEST_METRICS_BY_GROUP_TOPIC, clusterPhyId, group, topic, + DslConstant.LIST_GROUP_LATEST_METRICS_BY_GROUP_TOPIC, clusterPhyId, group, topic, startTime, latestTime, aggDsl); String routing = routing(clusterPhyId, group); @@ -86,7 +85,7 @@ public class GroupMetricESDAO extends BaseMetricESDAO { String realIndex = realIndex(startTime, latestTime); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.LIST_GROUP_LATEST_METRICS_OF_PARTITION, clusterPhyId, group, topic, latestTime); + DslConstant.LIST_GROUP_LATEST_METRICS_OF_PARTITION, clusterPhyId, group, topic, latestTime); List groupMetricPOS = esOpClient.performRequest(realIndex, dsl, GroupMetricPO.class); return filterMetrics(groupMetricPOS, metrics); @@ -101,8 +100,8 @@ public class GroupMetricESDAO extends BaseMetricESDAO { String matchDsl = buildTermsDsl(Arrays.asList(match)); String dsl = match.isEqual() - ? dslLoaderUtil.getFormatDslByFileName(DslsConstant.COUNT_GROUP_METRIC_VALUE, clusterPhyId, groupName, startTime, endTime, matchDsl) - : dslLoaderUtil.getFormatDslByFileName(DslsConstant.COUNT_GROUP_NOT_METRIC_VALUE, clusterPhyId, groupName, startTime, endTime, matchDsl); + ? dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_GROUP_METRIC_VALUE, clusterPhyId, groupName, startTime, endTime, matchDsl) + : dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_GROUP_NOT_METRIC_VALUE, clusterPhyId, groupName, startTime, endTime, matchDsl); return esOpClient.performRequestWithRouting(clusterPhyId.toString() + "@" + groupName, realIndex, dsl, s -> handleESQueryResponseCount(s), 3); @@ -127,7 +126,7 @@ public class GroupMetricESDAO extends BaseMetricESDAO { Integer partition = tp.getPartition(); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.LIST_GROUP_METRICS, clusterId, groupName, topic, partition, startTime, endTime, interval, aggDsl); + DslConstant.LIST_GROUP_METRICS, clusterId, groupName, topic, partition, startTime, endTime, interval, aggDsl); Map> metricMap = esOpClient.performRequest(realIndex, dsl, s -> handleGroupMetrics(s, aggType, metrics), 3); @@ -148,7 +147,7 @@ public class GroupMetricESDAO extends BaseMetricESDAO { String realIndex = realIndex(startTime, endTime); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_GROUP_TOPIC_PARTITION, clusterPhyId, groupName, startTime, endTime); + DslConstant.GET_GROUP_TOPIC_PARTITION, clusterPhyId, groupName, startTime, endTime); List groupMetricPOS = esOpClient.performRequestWithRouting(routing(clusterPhyId, groupName), realIndex, dsl, GroupMetricPO.class); return groupMetricPOS.stream().map(g -> new TopicPartitionKS(g.getTopic(), g.getPartitionId().intValue())).collect( Collectors.toSet()); diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/PartitionMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/PartitionMetricESDAO.java index 4f86852b..759f5eb5 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/PartitionMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/PartitionMetricESDAO.java @@ -1,14 +1,14 @@ package com.xiaojukeji.know.streaming.km.persistence.es.dao; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.PartitionMetricPO; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.util.List; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.PARTITION_INDEX; /** * @author didi @@ -18,10 +18,9 @@ public class PartitionMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = PARTITION_INDEX; - super.indexTemplate = PARTITION_TEMPLATE; + super.indexName = PARTITION_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register(this); } public PartitionMetricPO getPartitionLatestMetrics(Long clusterPhyId, String topic, @@ -31,7 +30,7 @@ public class PartitionMetricESDAO extends BaseMetricESDAO { Long startTime = endTime - FIVE_MIN; String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_PARTITION_LATEST_METRICS, clusterPhyId, topic, brokerId, partitionId, startTime, endTime); + DslConstant.GET_PARTITION_LATEST_METRICS, clusterPhyId, topic, brokerId, partitionId, startTime, endTime); PartitionMetricPO partitionMetricPO = esOpClient.performRequestAndTakeFirst( partitionId.toString(), realIndex(startTime, endTime), dsl, PartitionMetricPO.class); @@ -45,7 +44,7 @@ public class PartitionMetricESDAO extends BaseMetricESDAO { Long startTime = endTime - FIVE_MIN; String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.LIST_PARTITION_LATEST_METRICS_BY_TOPIC, clusterPhyId, topic, startTime, endTime); + DslConstant.LIST_PARTITION_LATEST_METRICS_BY_TOPIC, clusterPhyId, topic, startTime, endTime); List partitionMetricPOS = esOpClient.performRequest( realIndex(startTime, endTime), dsl, PartitionMetricPO.class); diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ReplicationMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ReplicationMetricESDAO.java index fbd874f2..6f1c7561 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ReplicationMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ReplicationMetricESDAO.java @@ -4,7 +4,7 @@ import com.didiglobal.logi.elasticsearch.client.response.query.query.ESQueryResp import com.didiglobal.logi.elasticsearch.client.response.query.query.aggs.ESAggr; import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.ReplicationMetricPO; import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -14,7 +14,7 @@ import java.util.List; import java.util.Map; import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.VALUE; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.REPLICATION_INDEX; /** * @author didi @@ -24,10 +24,9 @@ public class ReplicationMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = REPLICATION_INDEX; - super.indexTemplate = REPLICATION_TEMPLATE; + super.indexName = REPLICATION_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register(this); } /** @@ -39,7 +38,7 @@ public class ReplicationMetricESDAO extends BaseMetricESDAO { Long startTime = endTime - FIVE_MIN; String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_REPLICATION_LATEST_METRICS, clusterPhyId, brokerId, topic, partitionId, startTime, endTime); + DslConstant.GET_REPLICATION_LATEST_METRICS, clusterPhyId, brokerId, topic, partitionId, startTime, endTime); ReplicationMetricPO replicationMetricPO = esOpClient.performRequestAndTakeFirst( realIndex(startTime, endTime), dsl, ReplicationMetricPO.class); @@ -61,7 +60,7 @@ public class ReplicationMetricESDAO extends BaseMetricESDAO { String aggDsl = buildAggsDSL(metrics, aggType); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_REPLICATION_AGG_SINGLE_METRICS, clusterPhyId, brokerId,topic, partitionId, startTime, endTime, aggDsl); + DslConstant.GET_REPLICATION_AGG_SINGLE_METRICS, clusterPhyId, brokerId,topic, partitionId, startTime, endTime, aggDsl); return esOpClient.performRequestWithRouting(String.valueOf(brokerId), realIndex, dsl, s -> handleSingleESQueryResponse(s, metrics, aggType), 3); diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/TopicMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/TopicMetricESDAO.java index e997a778..49749160 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/TopicMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/TopicMetricESDAO.java @@ -13,7 +13,7 @@ import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPoint import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; import com.xiaojukeji.know.streaming.km.common.utils.Tuple; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @@ -22,17 +22,16 @@ import java.util.*; import java.util.stream.Collectors; import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.TOPIC_INDEX; @Component public class TopicMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = TOPIC_INDEX; - super.indexTemplate = TOPIC_TEMPLATE; + super.indexName = TOPIC_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register(this); } protected FutureWaitUtil queryFuture = FutureWaitUtil.init("TopicMetricESDAO", 4,8, 500); @@ -47,7 +46,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { String sortDsl = buildSortDsl(sort, SearchSort.DEFAULT); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_MAX_OR_MIN_SINGLE_METRIC, clusterPhyId, startTime, endTime, topic, sortDsl); + DslConstant.GET_TOPIC_MAX_OR_MIN_SINGLE_METRIC, clusterPhyId, startTime, endTime, topic, sortDsl); TopicMetricPO topicMetricPO = esOpClient.performRequestAndTakeFirst(topic, realIndex, dsl, TopicMetricPO.class); ret.add(topicMetricPO); } @@ -74,7 +73,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { } String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_AGG_SINGLE_METRICS, clusterPhyId, startTime, endTime, appendQueryDsl.toString(), aggDsl); + DslConstant.GET_TOPIC_AGG_SINGLE_METRICS, clusterPhyId, startTime, endTime, appendQueryDsl.toString(), aggDsl); return esOpClient.performRequest(realIndex, dsl, s -> handleSingleESQueryResponse(s, metrics, aggType), 3); @@ -112,7 +111,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { String realIndex = realIndex(startTime, latestMetricTime); String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.LIST_TOPIC_WITH_LATEST_METRICS, clusterId, latestMetricTime, appendQueryDsl.toString(), sortDsl); + DslConstant.LIST_TOPIC_WITH_LATEST_METRICS, clusterId, latestMetricTime, appendQueryDsl.toString(), sortDsl); return esOpClient.performRequest(realIndex, dsl, TopicMetricPO.class); } @@ -126,8 +125,8 @@ public class TopicMetricESDAO extends BaseMetricESDAO { String termDsl = buildTermsDsl(Arrays.asList(term)); String dsl = term.isEqual() - ? dslLoaderUtil.getFormatDslByFileName(DslsConstant.COUNT_TOPIC_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl) - : dslLoaderUtil.getFormatDslByFileName(DslsConstant.COUNT_TOPIC_NOT_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl); + ? dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_TOPIC_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl) + : dslLoaderUtil.getFormatDslByFileName( DslConstant.COUNT_TOPIC_NOT_METRIC_VALUE, clusterPhyId, topic, startTime, endTime, termDsl); return esOpClient.performRequestWithRouting(topic, realIndex, dsl, s -> handleESQueryResponseCount(s), 3); @@ -141,7 +140,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { Long startTime = endTime - FIVE_MIN; String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_BROKER_LATEST_METRICS, clusterPhyId, topic, brokerId, startTime, endTime); + DslConstant.GET_TOPIC_BROKER_LATEST_METRICS, clusterPhyId, topic, brokerId, startTime, endTime); TopicMetricPO topicMetricPO = esOpClient.performRequestAndTakeFirst(topic, realIndex(startTime, endTime), dsl, TopicMetricPO.class); @@ -165,7 +164,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { } String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_LATEST_METRICS, clusterPhyId, startTime, endTime, appendQueryDsl.toString()); + DslConstant.GET_TOPIC_LATEST_METRICS, clusterPhyId, startTime, endTime, appendQueryDsl.toString()); //topicMetricPOS 已经按照 timeStamp 倒序排好序了 List topicMetricPOS = esOpClient.performRequest(realIndex(startTime, endTime), dsl, TopicMetricPO.class); @@ -197,7 +196,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { } String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_LATEST_METRICS, clusterPhyId, startTime, endTime, appendQueryDsl.toString()); + DslConstant.GET_TOPIC_LATEST_METRICS, clusterPhyId, startTime, endTime, appendQueryDsl.toString()); TopicMetricPO topicMetricPO = esOpClient.performRequestAndTakeFirst(topic, realIndex(startTime, endTime), dsl, TopicMetricPO.class); @@ -262,7 +261,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { 3000, () -> { String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_AGG_LIST_METRICS, clusterPhyId, topic, startTime, endTime, interval, aggDsl); + DslConstant.GET_TOPIC_AGG_LIST_METRICS, clusterPhyId, topic, startTime, endTime, interval, aggDsl); Map> metricMap = esOpClient.performRequestWithRouting(topic, realIndex, dsl, s -> handleListESQueryResponse(s, metrics, aggType), 3); @@ -299,7 +298,7 @@ public class TopicMetricESDAO extends BaseMetricESDAO { //4、查询es String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_TOPIC_AGG_TOP_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); + DslConstant.GET_TOPIC_AGG_TOP_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); return esOpClient.performRequest(realIndex, dsl, s -> handleTopTopicESQueryResponse(s, metrics, topN), 3); diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ZookeeperMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ZookeeperMetricESDAO.java index dc19d2b8..b94a2461 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ZookeeperMetricESDAO.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/ZookeeperMetricESDAO.java @@ -5,7 +5,7 @@ import com.didiglobal.logi.elasticsearch.client.response.query.query.aggs.ESAggr import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; import com.xiaojukeji.know.streaming.km.common.constant.ESConstant; import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; -import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslsConstant; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -15,18 +15,16 @@ import java.util.List; import java.util.Map; import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.ZOOKEEPER_INDEX; -import static com.xiaojukeji.know.streaming.km.common.constant.ESIndexConstant.ZOOKEEPER_TEMPLATE; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.ZOOKEEPER_INDEX; @Component public class ZookeeperMetricESDAO extends BaseMetricESDAO { @PostConstruct public void init() { - super.indexName = ZOOKEEPER_INDEX; - super.indexTemplate = ZOOKEEPER_TEMPLATE; + super.indexName = ZOOKEEPER_INDEX; checkCurrentDayIndexExist(); - BaseMetricESDAO.register(indexName, this); + register(this); } /** @@ -49,7 +47,7 @@ public class ZookeeperMetricESDAO extends BaseMetricESDAO { //4、构造dsl查询条件，开始查询 try { String dsl = dslLoaderUtil.getFormatDslByFileName( - DslsConstant.GET_ZOOKEEPER_AGG_LIST_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); + DslConstant.GET_ZOOKEEPER_AGG_LIST_METRICS, clusterPhyId, startTime, endTime, interval, aggDsl); return esOpClient.performRequestWithRouting( String.valueOf(clusterPhyId), diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectClusterMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectClusterMetricESDAO.java new file mode 100644 index 00000000..07394f51 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectClusterMetricESDAO.java @@ -0,0 +1,275 @@ +package com.xiaojukeji.know.streaming.km.persistence.es.dao.connect; + +import com.didiglobal.logi.elasticsearch.client.response.query.query.ESQueryResponse; +import com.didiglobal.logi.elasticsearch.client.response.query.query.aggs.ESAggr; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; +import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; +import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; +import com.xiaojukeji.know.streaming.km.common.utils.Tuple; +import com.xiaojukeji.know.streaming.km.persistence.es.dao.BaseMetricESDAO; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import javax.annotation.PostConstruct; +import java.util.*; +import java.util.stream.Collectors; + +import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CONNECT_CLUSTER_INDEX; + +@Component +public class ConnectClusterMetricESDAO extends BaseMetricESDAO { + @PostConstruct + public void init() { + super.indexName = CONNECT_CLUSTER_INDEX; + checkCurrentDayIndexExist(); + register( this); + } + + protected FutureWaitUtil queryFuture = FutureWaitUtil.init("ConnectClusterMetricESDAO", 4,8, 500); + + /** + * 获取集群 clusterPhyId 中每个 metric 的 topN 的 connectCluster 在指定时间[startTime、endTime]区间内所有的指标 + * topN 按照[startTime, endTime] 时间段内最后一个值来排序 + */ + public Table> listMetricsByTop(Long clusterPhyId, + List connectClusterIdList, + List metricNameList, + String aggType, + int topN, + Long startTime, + Long endTime){ + // 1、获取TopN + Map> topNConnectClusterIds = getTopNConnectClusterIds(clusterPhyId, metricNameList, aggType, topN, startTime, endTime); + + Table> table = HashBasedTable.create(); + + // 2、查询指标 + for(String metric : metricNameList) { + table.putAll( + this.listMetricsByConnectClusterIdList( + clusterPhyId, + Arrays.asList(metric), + aggType, + topNConnectClusterIds.getOrDefault(metric, connectClusterIdList), + startTime, + endTime + ) + ); + } + + return table; + } + + /** + * 获取集群 clusterPhyId 中每个 metric 的指定 connectClusters 在指定时间[startTime、endTime]区间内所有的指标 + */ + public Table> listMetricsByConnectClusterIdList(Long clusterPhyId, + List metricNameList, + String aggType, + List connectClusterIdList, + Long startTime, + Long endTime){ + //1、获取需要查下的索引 + String realIndex = realIndex(startTime, endTime); + + //2、根据查询的时间区间大小来确定指标点的聚合区间大小 + String interval = MetricsUtils.getInterval(endTime - startTime); + + //3、构造agg查询条件 + String aggDsl = buildAggsDSL(metricNameList, aggType); + + final Table> table = HashBasedTable.create(); + + //4、构造dsl查询条件 + for(Long connectClusterId : connectClusterIdList){ + try { + String dsl = dslLoaderUtil.getFormatDslByFileName( + DslConstant.GET_CONNECT_CLUSTER_AGG_LIST_METRICS, + clusterPhyId, + connectClusterId, + startTime, + endTime, + interval, + aggDsl + ); + + queryFuture.runnableTask( + String.format("class=ConnectClusterMetricESDAO||method=listMetricsByConnectClusterIdList||ClusterPhyId=%d", clusterPhyId), + 5000, + () -> { + Map> metricMap = esOpClient.performRequestWithRouting( + String.valueOf(connectClusterId), + realIndex, + dsl, + s -> handleListESQueryResponse(s, metricNameList, aggType), + 3 + ); + + synchronized (table) { + for(String metric : metricMap.keySet()){ + table.put(metric, connectClusterId, metricMap.get(metric)); + } + } + }); + } catch (Exception e) { + LOGGER.error( + "class=ConnectClusterMetricESDAO||method=listMetricsByConnectClusterIdList||clusterPhyId={}||connectClusterId{}||errMsg=exception!", + clusterPhyId, connectClusterId, e + ); + } + } + + queryFuture.waitExecute(); + + return table; + } + + /** + * 获取集群 clusterPhyId 中每个 metric 的 topN 的 broker + */ + //public for test + public Map> getTopNConnectClusterIds(Long clusterPhyId, + List metricNameList, + String aggType, + int topN, + Long startTime, + Long endTime){ + //1、获取需要查下的索引 + String realIndex = realIndex(startTime, endTime); + + //2、根据查询的时间区间大小来确定指标点的聚合区间大小 + String interval = MetricsUtils.getInterval(endTime - startTime); + + //3、构造agg查询条件 + String aggDsl = buildAggsDSL(metricNameList, aggType); + + //4、查询es + String dsl = dslLoaderUtil.getFormatDslByFileName( + DslConstant.GET_CONNECT_CLUSTER_AGG_TOP_METRICS, + clusterPhyId, + startTime, + endTime, + interval, + aggDsl + ); + + return esOpClient.performRequest( + realIndex, + dsl, + s -> handleTopConnectClusterESQueryResponse(s, metricNameList, topN), + 3 + ); + } + + /**************************************************** private method ****************************************************/ + + private Map> handleListESQueryResponse(ESQueryResponse response, List metrics, String aggType){ + Map> metricMap = new HashMap<>(); + + if(null == response || null == response.getAggs()){ + return metricMap; + } + + Map esAggrMap = response.getAggs().getEsAggrMap(); + if (null == esAggrMap || null == esAggrMap.get(HIST)) { + return metricMap; + } + + if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){ + return metricMap; + } + + for(String metric : metrics){ + List metricPoints = new ArrayList<>(); + + esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { + try { + if (null != esBucket.getUnusedMap().get(KEY)) { + Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); + Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE); + if(null == value){return;} + + MetricPointVO metricPoint = new MetricPointVO(); + metricPoint.setAggType(aggType); + metricPoint.setTimeStamp(timestamp); + metricPoint.setValue(value.toString()); + metricPoint.setName(metric); + + metricPoints.add(metricPoint); + }else { + LOGGER.info(""); + } + }catch (Exception e){ + LOGGER.error("metric={}||errMsg=exception!", metric, e); + } + } ); + + metricMap.put(metric, optimizeMetricPoints(metricPoints)); + } + + return metricMap; + } + + private Map> handleTopConnectClusterESQueryResponse(ESQueryResponse response, List metrics, int topN){ + Map> ret = new HashMap<>(); + + if(null == response || null == response.getAggs()){ + return ret; + } + + Map esAggrMap = response.getAggs().getEsAggrMap(); + if (null == esAggrMap || null == esAggrMap.get(HIST)) { + return ret; + } + + if(CollectionUtils.isEmpty(esAggrMap.get(HIST).getBucketList())){ + return ret; + } + + Map>> metricBrokerValueMap = new HashMap<>(); + + //1、先获取每个指标对应的所有brokerIds以及指标的值 + for(String metric : metrics) { + esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { + try { + if (null != esBucket.getUnusedMap().get(KEY)) { + Long connectorClusterId = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); + Object value = esBucket.getAggrMap().get(HIST).getBucketList() + .get(0).getAggrMap().get(metric).getUnusedMap().get(VALUE); + + if(null == value){return;} + + List> connectorClusterValue = (null == metricBrokerValueMap.get(metric)) ? + new ArrayList<>() : metricBrokerValueMap.get(metric); + + connectorClusterValue.add(new Tuple<>(connectorClusterId, Double.valueOf(value.toString()))); + metricBrokerValueMap.put(metric, connectorClusterValue); + } + }catch (Exception e){ + LOGGER.error("metric={}||errMsg=exception!", metric, e); + } + } ); + } + + //2、对每个指标的broker按照指标值排序，并截取前topN个brokerIds + for(String metric : metricBrokerValueMap.keySet()){ + List> connectorClusterValue = metricBrokerValueMap.get(metric); + + connectorClusterValue.sort((o1, o2) -> { + if(null == o1 || null == o2){return 0;} + return o2.getV2().compareTo(o1.getV2()); + } ); + + List> temp = (connectorClusterValue.size() > topN) ? connectorClusterValue.subList(0, topN) : connectorClusterValue; + List connectorClusterIds = temp.stream().map(t -> t.getV1()).collect(Collectors.toList()); + + ret.put(metric, connectorClusterIds); + } + + return ret; + } +} diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectorMetricESDAO.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectorMetricESDAO.java new file mode 100644 index 00000000..060f57be --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dao/connect/ConnectorMetricESDAO.java @@ -0,0 +1,365 @@ +package com.xiaojukeji.know.streaming.km.persistence.es.dao.connect; + +import com.alibaba.druid.util.StringUtils; +import com.didiglobal.logi.elasticsearch.client.response.query.query.ESQueryResponse; +import com.didiglobal.logi.elasticsearch.client.response.query.query.aggs.ESAggr; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import com.xiaojukeji.know.streaming.km.common.bean.entity.search.SearchTerm; +import com.xiaojukeji.know.streaming.km.common.bean.po.metrice.connect.ConnectorMetricPO; +import com.xiaojukeji.know.streaming.km.common.bean.vo.metrics.point.MetricPointVO; +import com.xiaojukeji.know.streaming.km.common.utils.FutureWaitUtil; +import com.xiaojukeji.know.streaming.km.common.utils.MetricsUtils; +import com.xiaojukeji.know.streaming.km.common.utils.Triple; +import com.xiaojukeji.know.streaming.km.common.utils.Tuple; +import com.xiaojukeji.know.streaming.km.persistence.es.dao.BaseMetricESDAO; +import com.xiaojukeji.know.streaming.km.persistence.es.dsls.DslConstant; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; + +import static com.xiaojukeji.know.streaming.km.common.constant.ESConstant.*; +import static com.xiaojukeji.know.streaming.km.persistence.es.template.TemplateConstant.CONNECT_CONNECTOR_INDEX; + +@Component +public class ConnectorMetricESDAO extends BaseMetricESDAO { + + @PostConstruct + public void init() { + super.indexName = CONNECT_CONNECTOR_INDEX; + checkCurrentDayIndexExist(); + register( this); + } + + protected FutureWaitUtil queryFuture = FutureWaitUtil.init("ConnectorMetricESDAO", 4,8, 500); + + + /** + * 获取每个 metric 的 topN 个 connector 的指标，如果获取不到 topN 的topics, 则默认返回 defaultTopics 的指标 + */ + public Table, List> listMetricsByTopN(Long clusterPhyId, + List> defaultConnectorList, + List metricNameList, + String aggType, + int topN, + Long startTime, + Long endTime){ + //1、获取topN要查询的topic，每一个指标的topN的topic可能不一样 + Map>> metricsMap = this.getTopNConnectors(clusterPhyId, metricNameList, aggType, topN, startTime, endTime); + + Table, List> table = HashBasedTable.create(); + + for(String metricName : metricNameList){ + table.putAll(this.listMetricsByConnectors( + clusterPhyId, + Arrays.asList(metricName), + aggType, + metricsMap.getOrDefault(metricName, defaultConnectorList), + startTime, + endTime) + ); + } + + return table; + } + + public List getConnectorLatestMetric(Long clusterPhyId, List> connectClusterIdAndConnectorNameList, List metricsNames){ + List connectorMetricPOS = new CopyOnWriteArrayList<>(); + + for(Tuple connectClusterIdAndConnectorName : connectClusterIdAndConnectorNameList){ + queryFuture.runnableTask( + "getConnectorLatestMetric", + 30000, + () -> { + ConnectorMetricPO connectorMetricPO = this.getConnectorLatestMetric(clusterPhyId, connectClusterIdAndConnectorName.getV1(), connectClusterIdAndConnectorName.getV2(), metricsNames); + connectorMetricPOS.add(connectorMetricPO); + }); + } + + queryFuture.waitExecute(); + return connectorMetricPOS; + } + + public ConnectorMetricPO getConnectorLatestMetric(Long clusterPhyId, Long connectClusterId, String connectorName, List metricsNames){ + Long endTime = getLatestMetricTime(); + Long startTime = endTime - FIVE_MIN; + + SearchTerm searchClusterIdTerm = new SearchTerm("connectClusterId", connectClusterId.toString()); + searchClusterIdTerm.setField(true); + + SearchTerm searchClusterNameTerm = new SearchTerm("connectorName", connectorName); + searchClusterNameTerm.setField(true); + + String termDsl = buildTermsDsl(Arrays.asList(searchClusterIdTerm, searchClusterNameTerm)); + StringBuilder appendQueryDsl = new StringBuilder(); + if(!StringUtils.isEmpty(termDsl)){ + appendQueryDsl.append(",").append(termDsl); + } + + String dsl = dslLoaderUtil.getFormatDslByFileName( + DslConstant.GET_CONNECTOR_LATEST_METRICS, connectClusterId, connectorName, startTime, endTime, appendQueryDsl.toString()); + + ConnectorMetricPO connectorMetricPO = esOpClient.performRequestAndTakeFirst( + connectClusterId.toString(), realIndex(startTime, endTime), dsl, ConnectorMetricPO.class); + + return (null == connectorMetricPO) ? new ConnectorMetricPO(clusterPhyId, connectClusterId, connectorName) + : filterMetrics(connectorMetricPO, metricsNames); + } + + /** + * 获取每个 metric 指定个 topic 的指标 + */ + public Table, List> listMetricsByConnectors(Long clusterPhyId, + List metrics, + String aggType, + List> connectorList, + Long startTime, + Long endTime) { + //1、获取需要查下的索引 + String realIndex = realIndex(startTime, endTime); + + //2、根据查询的时间区间大小来确定指标点的聚合区间大小 + String interval = MetricsUtils.getInterval(endTime - startTime); + + //3、构造agg查询条件 + String aggDsl = buildAggsDSL(metrics, aggType); + + final Table, List> table = HashBasedTable.create(); + + //4、构造dsl查询条件 + for(Tuple connector : connectorList) { + try { + queryFuture.runnableTask( + String.format( + "method=listConnectorMetricsByConnectors||ClusterPhyId=%d||connectorName=%s", + clusterPhyId, connector.getV2() ), + 3000, + () -> { + String dsl = dslLoaderUtil.getFormatDslByFileName( + DslConstant.GET_CONNECTOR_AGG_LIST_METRICS, + clusterPhyId, + connector.getV1(), + connector.getV2(), + startTime, + endTime, + interval, + aggDsl + ); + + Map> metricMap = esOpClient.performRequestWithRouting( + connector.getV1().toString(), + realIndex, + dsl, + s -> handleListESQueryResponse(s, metrics, aggType), + 3 + ); + + synchronized (table){ + for(String metric : metricMap.keySet()){ + table.put(metric, connector, metricMap.get(metric)); + } + } + }); + } catch (Exception e) { + LOGGER.error( + "method=listConnectorMetricsByConnectors||clusterPhyId={}||connectorName{}||errMsg=exception!", + clusterPhyId, connector.getV2(), e + ); + } + } + + queryFuture.waitExecute(); + + return table; + } + + //public for test + public Map>> getTopNConnectors(Long clusterPhyId, + List metricNameList, + String aggType, + int topN, + Long startTime, + Long endTime){ + //1、获取需要查下的索引 + String realIndex = realIndex(startTime, endTime); + + //2、根据查询的时间区间大小来确定指标点的聚合区间大小 + String interval = MetricsUtils.getInterval(endTime - startTime); + + //3、构造agg查询条件 + String aggDsl = buildAggsDSL(metricNameList, aggType); + + //4、查询es + String dsl = dslLoaderUtil.getFormatDslByFileName( + DslConstant.GET_CONNECTOR_AGG_TOP_METRICS, + clusterPhyId, + startTime, + endTime, + interval, + aggDsl + ); + + return esOpClient.performRequest( + realIndex, + dsl, + s -> handleTopConnectorESQueryResponse(s, metricNameList, topN), + 3 + ); + } + + /**************************************************** private method ****************************************************/ + + private Table handleSingleESQueryResponse(ESQueryResponse response, List metrics, String aggType){ + Table table = HashBasedTable.create(); + + Map esAggrMap = checkBucketsAndHitsOfResponseAggs(response); + if(null == esAggrMap){return table;} + + for(String metric : metrics){ + esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { + try { + if (null != esBucket.getUnusedMap().get(KEY)) { + String topic = esBucket.getUnusedMap().get(KEY).toString(); + String value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE).toString(); + + MetricPointVO metricPoint = new MetricPointVO(); + metricPoint.setAggType(aggType); + metricPoint.setValue(value); + metricPoint.setName(metric); + + table.put(topic, metric, metricPoint); + }else { + LOGGER.debug("method=handleListESQueryResponse||metric={}||errMsg=get topic is null!", metric); + } + }catch (Exception e){ + LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e); + } + }); + } + + return table; + } + + private Map> handleListESQueryResponse(ESQueryResponse response, List metrics, String aggType){ + Map> metricMap = new HashMap<>(); + + Map esAggrMap = checkBucketsAndHitsOfResponseAggs(response); + if(null == esAggrMap){return metricMap;} + + for(String metric : metrics){ + List metricPoints = new ArrayList<>(); + + esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { + try { + if (null != esBucket.getUnusedMap().get(KEY)) { + Long timestamp = Long.valueOf(esBucket.getUnusedMap().get(KEY).toString()); + Object value = esBucket.getAggrMap().get(metric).getUnusedMap().get(VALUE); + if(value == null){return;} + + MetricPointVO metricPoint = new MetricPointVO(); + metricPoint.setAggType(aggType); + metricPoint.setTimeStamp(timestamp); + metricPoint.setValue(value.toString()); + metricPoint.setName(metric); + + metricPoints.add(metricPoint); + }else { + LOGGER.info(""); + } + }catch (Exception e){ + LOGGER.error("method=handleListESQueryResponse||metric={}||errMsg=exception!", metric, e); + } + } ); + + metricMap.put(metric, optimizeMetricPoints(metricPoints)); + } + + return metricMap; + } + + private Map>> handleTopConnectorESQueryResponse(ESQueryResponse response, List metricNameList, int topN){ + Map>> ret = new HashMap<>(); + + Map esAggrMap = checkBucketsAndHitsOfResponseAggs(response); + if(null == esAggrMap) { + return ret; + } + + Map>> metricValueMap = new HashMap<>(); + + // 1、先获取每个指标对应的所有 connector 以及指标的值 + for(String metricName : metricNameList) { + esAggrMap.get(HIST).getBucketList().forEach( esBucket -> { + try { + if (null != esBucket.getUnusedMap().get(KEY)) { + String connectorNameAndClusterId = esBucket.getUnusedMap().get(KEY).toString(); + Object value = esBucket.getAggrMap().get(HIST).getBucketList().get(0).getAggrMap().get(metricName).getUnusedMap().get(VALUE); + if (value == null) { + return; + } + Double metricValue = Double.valueOf(value.toString()); + + Tuple tuple = splitConnectorNameAndClusterId(connectorNameAndClusterId); + if (null == tuple) { + return; + } + + metricValueMap.putIfAbsent(metricName, new ArrayList<>()); + metricValueMap.get(metricName).add(new Triple<>(tuple.getV2(), tuple.getV1(), metricValue)); + } + } catch (Exception e) { + LOGGER.error("method=handleTopConnectorESQueryResponse||metricName={}||errMsg=exception!", metricName, e); + } + } ); + } + + //2、对每个指标的connector按照指标值排序，并截取前topN个connectors + for(Map.Entry>> entry : metricValueMap.entrySet()){ + entry.getValue().sort((o1, o2) -> { + if(null == o1 || null == o2) { + return 0; + } + + return o2.v3().compareTo(o1.v3()); + } ); + + List> temp = (entry.getValue().size() > topN) ? entry.getValue().subList(0, topN) : entry.getValue(); + + List> connectorList = new ArrayList<>(); + for (Triple triple: temp) { + connectorList.add(new Tuple<>(triple.v1(), triple.v2())); + } + + ret.put(entry.getKey(), connectorList); + } + + return ret; + } + + private Map>> topicMetricMap2MetricTopicMap( + Map>> topicMetricMap){ + Map>> ret = new HashMap<>(); + + for(String topic : topicMetricMap.keySet()){ + Map> metricMap = topicMetricMap.get(topic); + + for(String metric : metricMap.keySet()){ + Map> brokerMap = (null == ret.get(metric)) ? new HashMap<>() : ret.get(metric); + + brokerMap.put(topic, metricMap.get(metric)); + ret.put(metric, brokerMap); + } + } + + return ret; + } + + private Tuple splitConnectorNameAndClusterId(String connectorNameAndClusterId){ + String[] ss = connectorNameAndClusterId.split("#"); + if(null == ss || ss.length != 2){return null;} + + return new Tuple<>(ss[0], Long.valueOf(ss[1])); + } +} diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslsConstant.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslConstant.java similarity index 83% rename from km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslsConstant.java rename to km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslConstant.java index f15fe7a8..9bd8062a 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslsConstant.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslConstant.java @@ -13,9 +13,9 @@ package com.xiaojukeji.know.streaming.km.persistence.es.dsls; * 在dslFiles目录下新建以类名为名称的文件夹，以方法名为名称的文件名 * */ -public class DslsConstant { +public class DslConstant { - private DslsConstant() {} + private DslConstant() {} /**************************************************** Base ****************************************************/ public static final String GET_LATEST_METRIC_TIME = "BaseMetricESDAO/getLatestMetricTime"; @@ -82,4 +82,18 @@ public class DslsConstant { /**************************************************** Zookeeper ****************************************************/ public static final String GET_ZOOKEEPER_AGG_LIST_METRICS = "ZookeeperMetricESDAO/getAggListZookeeperMetrics"; + + /**************************************************** Connect-Cluster ****************************************************/ + public static final String GET_CONNECT_CLUSTER_AGG_LIST_METRICS = "ConnectClusterMetricESDAO/getAggListConnectClusterMetrics"; + + public static final String GET_CONNECT_CLUSTER_AGG_TOP_METRICS = "ConnectClusterMetricESDAO/getAggTopMetricsConnectClusters"; + + /**************************************************** Connect-Connector ****************************************************/ + public static final String GET_CONNECTOR_LATEST_METRICS = "ConnectorMetricESDAO/getConnectorLatestMetric"; + + public static final String GET_CONNECTOR_AGG_LIST_METRICS = "ConnectorMetricESDAO/getConnectorAggListMetric"; + + public static final String GET_CONNECTOR_AGG_TOP_METRICS = "ConnectorMetricESDAO/getConnectorAggTopMetric"; + + } diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslLoaderUtil.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslLoaderUtil.java index 6bdea995..510b2def 100644 --- a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslLoaderUtil.java +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/dsls/DslLoaderUtil.java @@ -1,26 +1,14 @@ package com.xiaojukeji.know.streaming.km.persistence.es.dsls; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.parser.DefaultJSONParser; -import com.alibaba.fastjson.parser.Feature; -import com.alibaba.fastjson.parser.ParserConfig; -import com.alibaba.fastjson.serializer.SerializerFeature; import com.didiglobal.logi.log.ILog; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.xiaojukeji.know.streaming.km.common.utils.EnvUtil; import com.xiaojukeji.know.streaming.km.common.utils.LoggerUtil; +import com.xiaojukeji.know.streaming.km.persistence.es.ESFileLoader; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.Field; -import java.util.List; import java.util.Map; /** @@ -33,50 +21,25 @@ import java.util.Map; * */ @Component -public class DslLoaderUtil { +public class DslLoaderUtil extends ESFileLoader { private static final ILog LOGGER = LoggerUtil.getESLogger(); + + private static final String FILE_PATH = "es/dsl/"; + /** * 查询语句容器 + * key : fileRelativePath + * value : dslContent */ - private Map dslsMap = Maps.newHashMap(); + private Map dslsMap = Maps.newHashMap(); @PostConstruct public void init() { - LOGGER.info("method=init||DslLoaderUtil init start."); - List dslFileNames = Lists.newLinkedList(); - - // 反射获取接口中定义的变量中的值 - Field[] fields = DslsConstant.class.getDeclaredFields(); - for (int i = 0; i < fields.length; ++i) { - fields[i].setAccessible(true); - try { - dslFileNames.add(fields[i].get(null).toString()); - } catch (IllegalAccessException e) { - LOGGER.error("method=init||errMsg=fail to read {} error. ", fields[i].getName(), - e); - } - } - - // 加载dsl文件及内容 - for (String fileName : dslFileNames) { - dslsMap.put(fileName, readDslFileInJarFile(fileName)); - } - - // 输出加载的查询语句 - LOGGER.info("method=init||msg=dsl files count {}", dslsMap.size()); - for (Map.Entry entry : dslsMap.entrySet()) { - LOGGER.info("method=init||msg=file name {}, dsl content {}", entry.getKey(), - entry.getValue()); - } - - LOGGER.info("method=init||DslLoaderUtil init finished."); + dslsMap.putAll(loaderFileContext(FILE_PATH, DslConstant.class.getDeclaredFields())); } /** * 获取查询语句 - * - * @param fileName - * @return */ public String getDslByFileName(String fileName) { return dslsMap.get(fileName); @@ -84,10 +47,6 @@ public class DslLoaderUtil { /** * 获取格式化的查询语句 - * - * @param fileName - * @param args - * @return */ public String getFormatDslByFileName(String fileName, Object... args) { String loadDslContent = getDslByFileName(fileName); @@ -107,128 +66,4 @@ public class DslLoaderUtil { return dsl; } - - public String getFormatDslForCatIndexByCondition(String fileName, String boolMustDsl, Object... args) { - String formatDslByFileName = getFormatDslByFileName(fileName, args); - - return formatDslByFileName.replace("\"boolMustDsl\"", boolMustDsl); - } - - public String getFormatDslByFileNameByAggParam(String fileName, String clusterPhyMetrics, String interval, - String aggType, Object... args) { - String formatDslByFileName = getFormatDslByFileName(fileName, args); - - return formatDslByFileName - .replace("{interval}", interval) - .replace("{clusterPhyMetrics}", clusterPhyMetrics) - .replace("{aggType}", aggType); - } - - public String getFormatDslByFileNameAndOtherParam(String fileName, String interval, String aggsDsl, - Object... args) { - String formatDslByFileName = getFormatDslByFileName(fileName, args); - return formatDslByFileName - .replace("{interval}", interval) - .replace("\"aggsDsl\":1", aggsDsl); - } - - - public String getDslByTopNNameInfo(String fileName, String interval, String topNameStr, String aggsDsl, - Object... args) { - String formatDslByFileName = getFormatDslByFileName(fileName, args); - return formatDslByFileName - .replace("{interval}", interval) - .replace("\"aggsDsl\":1", aggsDsl) - .replace("\"topNameListStr\"", topNameStr); - } - - /**************************************************** private method ****************************************************/ - /** - * 去除json中的空格 - * - * @param sourceDsl - * @return - */ - private String trimJsonBank(String sourceDsl) { - List dslList = Lists.newArrayList(); - - DefaultJSONParser parser = null; - Object obj = null; - String dsl = sourceDsl; - - // 解析多个json，直到pos为0 - for (;;) { - try { - // 这里需要Feature.OrderedField.getMask()保持有序 - parser = new DefaultJSONParser(dsl, ParserConfig.getGlobalInstance(), - JSON.DEFAULT_PARSER_FEATURE | Feature.OrderedField.getMask()); - obj = parser.parse(); - } catch (Exception t) { - LOGGER.error("method=trimJsonBank||errMsg=parse json {} error. ", dsl, t); - } - if (obj == null) { - break; - } - - if (obj instanceof JSONObject) { - dslList.add( JSON.toJSONString(obj, SerializerFeature.WriteMapNullValue)); - int pos = parser.getLexer().pos(); - if (pos <= 0) { - break; - } - dsl = dsl.substring(pos); - parser.getLexer().close(); - } else { - parser.getLexer().close(); - break; - } - } - - // 格式化异常或者有多个查询语句，返回原来的查询语句 - if (dslList.isEmpty() || dslList.size() > 1) { - return sourceDsl; - } - - return dslList.get(0); - } - - /** - * 从jar包中读取dsl语句文件 - * - * @param fileName - * @return - */ - private String readDslFileInJarFile(String fileName) { - InputStream inputStream = this.getClass().getClassLoader() - .getResourceAsStream( String.format("dsl/%s", fileName)); - if (inputStream != null) { - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); - String line = null; - List lines = Lists.newLinkedList(); - try { - while ((line = bufferedReader.readLine()) != null) { - lines.add(line); - } - return StringUtils.join(lines, ""); - - } catch (IOException e) { - LOGGER.error("method=readDslFileInJarFile||errMsg=read file {} error. ", fileName, - e); - - return ""; - } finally { - try { - inputStream.close(); - } catch (IOException e) { - LOGGER.error( - "method=readDslFileInJarFile||errMsg=fail to close file {} error. ", - fileName, e); - } - } - } else { - LOGGER.error("method=readDslFileInJarFile||errMsg=fail to read file {} content", - fileName); - return ""; - } - } } diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateConstant.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateConstant.java new file mode 100644 index 00000000..2fc61f38 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateConstant.java @@ -0,0 +1,19 @@ +package com.xiaojukeji.know.streaming.km.persistence.es.template; + +/** + * @author didi + */ +public class TemplateConstant { + public static final String TOPIC_INDEX = "ks_kafka_topic_metric"; + public static final String CLUSTER_INDEX = "ks_kafka_cluster_metric"; + public static final String BROKER_INDEX = "ks_kafka_broker_metric"; + public static final String PARTITION_INDEX = "ks_kafka_partition_metric"; + public static final String GROUP_INDEX = "ks_kafka_group_metric"; + public static final String REPLICATION_INDEX = "ks_kafka_replication_metric"; + public static final String ZOOKEEPER_INDEX = "ks_kafka_zookeeper_metric"; + public static final String CONNECT_CLUSTER_INDEX = "ks_kafka_connect_cluster_metric"; + public static final String CONNECT_CONNECTOR_INDEX = "ks_kafka_connect_connector_metric"; + + private TemplateConstant() { + } +} diff --git a/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateLoaderUtil.java b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateLoaderUtil.java new file mode 100644 index 00000000..092506e7 --- /dev/null +++ b/km-persistence/src/main/java/com/xiaojukeji/know/streaming/km/persistence/es/template/TemplateLoaderUtil.java @@ -0,0 +1,28 @@ +package com.xiaojukeji.know.streaming.km.persistence.es.template; + +import com.google.common.collect.Maps; +import com.xiaojukeji.know.streaming.km.persistence.es.ESFileLoader; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.Map; + +@Component +public class TemplateLoaderUtil extends ESFileLoader { + + private static final String FILE_PATH = "es/template/"; + + /** + * 查询语句容器 + */ + private Map templateMap = Maps.newHashMap(); + + @PostConstruct + public void init() { + templateMap.putAll(loaderFileContext(FILE_PATH, TemplateConstant.class.getDeclaredFields())); + } + + public String getContextByFileName(String fileName) { + return templateMap.get(fileName); + } +} diff --git a/km-persistence/src/main/resources/dsl/BaseMetricESDAO/getLatestMetricTime b/km-persistence/src/main/resources/es/dsl/BaseMetricESDAO/getLatestMetricTime similarity index 100% rename from km-persistence/src/main/resources/dsl/BaseMetricESDAO/getLatestMetricTime rename to km-persistence/src/main/resources/es/dsl/BaseMetricESDAO/getLatestMetricTime diff --git a/km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getAggListBrokerMetrics b/km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getAggListBrokerMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getAggListBrokerMetrics rename to km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getAggListBrokerMetrics diff --git a/km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getAggSingleBrokerMetrics b/km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getAggSingleBrokerMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getAggSingleBrokerMetrics rename to km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getAggSingleBrokerMetrics diff --git a/km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getAggTopMetricsBrokers b/km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getAggTopMetricsBrokers similarity index 100% rename from km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getAggTopMetricsBrokers rename to km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getAggTopMetricsBrokers diff --git a/km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getBrokerLatestMetrics b/km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getBrokerLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/BrokerMetricESDAO/getBrokerLatestMetrics rename to km-persistence/src/main/resources/es/dsl/BrokerMetricESDAO/getBrokerLatestMetrics diff --git a/km-persistence/src/main/resources/dsl/ClusterMetricESDAO/getAggListClusterMetrics b/km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/getAggListClusterMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ClusterMetricESDAO/getAggListClusterMetrics rename to km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/getAggListClusterMetrics diff --git a/km-persistence/src/main/resources/dsl/ClusterMetricESDAO/getAggSingleClusterMetrics b/km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/getAggSingleClusterMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ClusterMetricESDAO/getAggSingleClusterMetrics rename to km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/getAggSingleClusterMetrics diff --git a/km-persistence/src/main/resources/dsl/ClusterMetricESDAO/getClusterLatestMetrics b/km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/getClusterLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ClusterMetricESDAO/getClusterLatestMetrics rename to km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/getClusterLatestMetrics diff --git a/km-persistence/src/main/resources/dsl/ClusterMetricESDAO/listClusterWithLatestMetrics b/km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/listClusterWithLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ClusterMetricESDAO/listClusterWithLatestMetrics rename to km-persistence/src/main/resources/es/dsl/ClusterMetricESDAO/listClusterWithLatestMetrics diff --git a/km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggListConnectClusterMetrics b/km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggListConnectClusterMetrics new file mode 100644 index 00000000..ebdb7dae --- /dev/null +++ b/km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggListConnectClusterMetrics @@ -0,0 +1,44 @@ +{ + "size": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "clusterPhyId": { + "value": %d + } + } + }, + { + "term": { + "connectClusterId": { + "value": %d + } + } + }, + { + "range": { + "timestamp": { + "gte": %d, + "lte": %d + } + } + } + ] + } + }, + "aggs": { + "hist": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "%s", + "time_zone": "Asia/Shanghai", + "min_doc_count": 0 + }, + "aggs": { + %s + } + } + } +} \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggTopMetricsConnectClusters b/km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggTopMetricsConnectClusters new file mode 100644 index 00000000..d3f64d83 --- /dev/null +++ b/km-persistence/src/main/resources/es/dsl/ConnectClusterMetricESDAO/getAggTopMetricsConnectClusters @@ -0,0 +1,45 @@ +{ + "size": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "clusterPhyId": { + "value": %d + } + } + }, + { + "range": { + "timestamp": { + "gte": %d, + "lte": %d + } + } + } + ] + } + }, + "aggs": { + "hist": { + "terms": { + "field": "connectClusterId", + "collect_mode": "breadth_first" + }, + "aggs": { + "hist": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "%s", + "time_zone": "Asia/Shanghai", + "min_doc_count": 0 + }, + "aggs": { + %s + } + } + } + } + } +} \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggListMetric b/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggListMetric new file mode 100644 index 00000000..f46c75f5 --- /dev/null +++ b/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggListMetric @@ -0,0 +1,51 @@ +{ + "size": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "clusterPhyId": { + "value": %d + } + } + }, + { + "term": { + "connectClusterId": { + "value": %d + } + } + }, + { + "term": { + "connectorName": { + "value": "%s" + } + } + }, + { + "range": { + "timestamp": { + "gte": %d, + "lte": %d + } + } + } + ] + } + }, + "aggs": { + "hist": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "%s", + "time_zone": "Asia/Shanghai", + "min_doc_count": 0 + }, + "aggs": { + %s + } + } + } +} \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggTopMetric b/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggTopMetric new file mode 100644 index 00000000..4dc1643b --- /dev/null +++ b/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorAggTopMetric @@ -0,0 +1,45 @@ +{ + "size": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "clusterPhyId": { + "value": %d + } + } + }, + { + "range": { + "timestamp": { + "gte": %d, + "lte": %d + } + } + } + ] + } + }, + "aggs": { + "hist": { + "terms": { + "field": "connectorNameAndClusterId", + "collect_mode": "breadth_first" + }, + "aggs": { + "hist": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "%s", + "time_zone": "Asia/Shanghai", + "min_doc_count": 0 + }, + "aggs": { + %s + } + } + } + } + } +} \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorLatestMetric b/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorLatestMetric new file mode 100644 index 00000000..ab1e16eb --- /dev/null +++ b/km-persistence/src/main/resources/es/dsl/ConnectorMetricESDAO/getConnectorLatestMetric @@ -0,0 +1,39 @@ +{ + "size":1000, + "query": { + "bool": { + "must": [ + { + "term": { + "connectClusterId": { + "value": %d + } + } + }, + { + "term": { + "connectorName": { + "value": "%s" + } + } + }, + { + "range": { + "timestamp": { + "gte": %d, + "lte": %d + } + } + } + %s + ] + } + }, + "sort": [ + { + "timestamp": { + "order": "desc" + } + } + ] +} \ No newline at end of file diff --git a/km-persistence/src/main/resources/dsl/GroupMetricESDAO/countGroupMetricValue b/km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/countGroupMetricValue similarity index 100% rename from km-persistence/src/main/resources/dsl/GroupMetricESDAO/countGroupMetricValue rename to km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/countGroupMetricValue diff --git a/km-persistence/src/main/resources/dsl/GroupMetricESDAO/countGroupNotMetricValue b/km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/countGroupNotMetricValue similarity index 100% rename from km-persistence/src/main/resources/dsl/GroupMetricESDAO/countGroupNotMetricValue rename to km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/countGroupNotMetricValue diff --git a/km-persistence/src/main/resources/dsl/GroupMetricESDAO/getTopicPartitionOfGroup b/km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/getTopicPartitionOfGroup similarity index 100% rename from km-persistence/src/main/resources/dsl/GroupMetricESDAO/getTopicPartitionOfGroup rename to km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/getTopicPartitionOfGroup diff --git a/km-persistence/src/main/resources/dsl/GroupMetricESDAO/listGroupMetrics b/km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/listGroupMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/GroupMetricESDAO/listGroupMetrics rename to km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/listGroupMetrics diff --git a/km-persistence/src/main/resources/dsl/GroupMetricESDAO/listLatestMetricsAggByGroupTopic b/km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/listLatestMetricsAggByGroupTopic similarity index 100% rename from km-persistence/src/main/resources/dsl/GroupMetricESDAO/listLatestMetricsAggByGroupTopic rename to km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/listLatestMetricsAggByGroupTopic diff --git a/km-persistence/src/main/resources/dsl/GroupMetricESDAO/listPartitionLatestMetrics b/km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/listPartitionLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/GroupMetricESDAO/listPartitionLatestMetrics rename to km-persistence/src/main/resources/es/dsl/GroupMetricESDAO/listPartitionLatestMetrics diff --git a/km-persistence/src/main/resources/dsl/PartitionMetricESDAO/getPartitionLatestMetrics b/km-persistence/src/main/resources/es/dsl/PartitionMetricESDAO/getPartitionLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/PartitionMetricESDAO/getPartitionLatestMetrics rename to km-persistence/src/main/resources/es/dsl/PartitionMetricESDAO/getPartitionLatestMetrics diff --git a/km-persistence/src/main/resources/dsl/PartitionMetricESDAO/listPartitionLatestMetricsByTopic b/km-persistence/src/main/resources/es/dsl/PartitionMetricESDAO/listPartitionLatestMetricsByTopic similarity index 100% rename from km-persistence/src/main/resources/dsl/PartitionMetricESDAO/listPartitionLatestMetricsByTopic rename to km-persistence/src/main/resources/es/dsl/PartitionMetricESDAO/listPartitionLatestMetricsByTopic diff --git a/km-persistence/src/main/resources/dsl/ReplicationMetricESDAO/getAggSingleReplicationMetrics b/km-persistence/src/main/resources/es/dsl/ReplicationMetricESDAO/getAggSingleReplicationMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ReplicationMetricESDAO/getAggSingleReplicationMetrics rename to km-persistence/src/main/resources/es/dsl/ReplicationMetricESDAO/getAggSingleReplicationMetrics diff --git a/km-persistence/src/main/resources/dsl/ReplicationMetricESDAO/getReplicationLatestMetrics b/km-persistence/src/main/resources/es/dsl/ReplicationMetricESDAO/getReplicationLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ReplicationMetricESDAO/getReplicationLatestMetrics rename to km-persistence/src/main/resources/es/dsl/ReplicationMetricESDAO/getReplicationLatestMetrics diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/countTopicMetricValue b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/countTopicMetricValue similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/countTopicMetricValue rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/countTopicMetricValue diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/countTopicNotMetricValue b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/countTopicNotMetricValue similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/countTopicNotMetricValue rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/countTopicNotMetricValue diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/getAggListMetrics b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggListMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/getAggListMetrics rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggListMetrics diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/getAggSingleMetrics b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggSingleMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/getAggSingleMetrics rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggSingleMetrics diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/getAggTopMetricsTopics b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggTopMetricsTopics similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/getAggTopMetricsTopics rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getAggTopMetricsTopics diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/getMaxOrMinSingleMetric b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getMaxOrMinSingleMetric similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/getMaxOrMinSingleMetric rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getMaxOrMinSingleMetric diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/getTopicLatestMetric b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getTopicLatestMetric similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/getTopicLatestMetric rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getTopicLatestMetric diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/getTopicLatestMetricByBrokerId b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getTopicLatestMetricByBrokerId similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/getTopicLatestMetricByBrokerId rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/getTopicLatestMetricByBrokerId diff --git a/km-persistence/src/main/resources/dsl/TopicMetricESDAO/listTopicWithLatestMetrics b/km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/listTopicWithLatestMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/TopicMetricESDAO/listTopicWithLatestMetrics rename to km-persistence/src/main/resources/es/dsl/TopicMetricESDAO/listTopicWithLatestMetrics diff --git a/km-persistence/src/main/resources/dsl/ZookeeperMetricESDAO/getAggListZookeeperMetrics b/km-persistence/src/main/resources/es/dsl/ZookeeperMetricESDAO/getAggListZookeeperMetrics similarity index 100% rename from km-persistence/src/main/resources/dsl/ZookeeperMetricESDAO/getAggListZookeeperMetrics rename to km-persistence/src/main/resources/es/dsl/ZookeeperMetricESDAO/getAggListZookeeperMetrics diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_broker_metric b/km-persistence/src/main/resources/es/template/ks_kafka_broker_metric new file mode 100644 index 00000000..749b1494 --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_broker_metric @@ -0,0 +1,101 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_broker_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "NetworkProcessorAvgIdle" : { + "type" : "float" + }, + "UnderReplicatedPartitions" : { + "type" : "float" + }, + "BytesIn_min_15" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "RequestHandlerAvgIdle" : { + "type" : "float" + }, + "connectionsCount" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "TotalRequestQueueSize" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "TotalResponseQueueSize" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_cluster_metric b/km-persistence/src/main/resources/es/template/ks_kafka_cluster_metric new file mode 100644 index 00000000..942fdce2 --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_cluster_metric @@ -0,0 +1,186 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "Connections" : { + "type" : "double" + }, + "BytesIn_min_15" : { + "type" : "double" + }, + "PartitionURP" : { + "type" : "double" + }, + "HealthScore_Topics" : { + "type" : "double" + }, + "EventQueueSize" : { + "type" : "double" + }, + "ActiveControllerCount" : { + "type" : "double" + }, + "GroupDeads" : { + "type" : "double" + }, + "BytesIn_min_5" : { + "type" : "double" + }, + "HealthCheckTotal_Topics" : { + "type" : "double" + }, + "Partitions" : { + "type" : "double" + }, + "BytesOut" : { + "type" : "double" + }, + "Groups" : { + "type" : "double" + }, + "BytesOut_min_15" : { + "type" : "double" + }, + "TotalRequestQueueSize" : { + "type" : "double" + }, + "HealthCheckPassed_Groups" : { + "type" : "double" + }, + "TotalProduceRequests" : { + "type" : "double" + }, + "HealthCheckPassed" : { + "type" : "double" + }, + "TotalLogSize" : { + "type" : "double" + }, + "GroupEmptys" : { + "type" : "double" + }, + "PartitionNoLeader" : { + "type" : "double" + }, + "HealthScore_Brokers" : { + "type" : "double" + }, + "Messages" : { + "type" : "double" + }, + "Topics" : { + "type" : "double" + }, + "PartitionMinISR_E" : { + "type" : "double" + }, + "HealthCheckTotal" : { + "type" : "double" + }, + "Brokers" : { + "type" : "double" + }, + "Replicas" : { + "type" : "double" + }, + "HealthCheckTotal_Groups" : { + "type" : "double" + }, + "GroupRebalances" : { + "type" : "double" + }, + "MessageIn" : { + "type" : "double" + }, + "HealthScore" : { + "type" : "double" + }, + "HealthCheckPassed_Topics" : { + "type" : "double" + }, + "HealthCheckTotal_Brokers" : { + "type" : "double" + }, + "PartitionMinISR_S" : { + "type" : "double" + }, + "BytesIn" : { + "type" : "double" + }, + "BytesOut_min_5" : { + "type" : "double" + }, + "GroupActives" : { + "type" : "double" + }, + "MessagesIn" : { + "type" : "double" + }, + "GroupReBalances" : { + "type" : "double" + }, + "HealthCheckPassed_Brokers" : { + "type" : "double" + }, + "HealthScore_Groups" : { + "type" : "double" + }, + "TotalResponseQueueSize" : { + "type" : "double" + }, + "Zookeepers" : { + "type" : "double" + }, + "LeaderMessages" : { + "type" : "double" + }, + "HealthScore_Cluster" : { + "type" : "double" + }, + "HealthCheckPassed_Cluster" : { + "type" : "double" + }, + "HealthCheckTotal_Cluster" : { + "type" : "double" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "type" : "date" + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_connect_cluster_metric b/km-persistence/src/main/resources/es/template/ks_kafka_connect_cluster_metric new file mode 100644 index 00000000..7fa4c523 --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_connect_cluster_metric @@ -0,0 +1,86 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_connect_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "connectClusterId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "ConnectorCount" : { + "type" : "float" + }, + "TaskCount" : { + "type" : "float" + }, + "ConnectorStartupAttemptsTotal" : { + "type" : "float" + }, + "ConnectorStartupFailurePercentage" : { + "type" : "float" + }, + "ConnectorStartupFailureTotal" : { + "type" : "float" + }, + "ConnectorStartupSuccessPercentage" : { + "type" : "float" + }, + "ConnectorStartupSuccessTotal" : { + "type" : "float" + }, + "TaskStartupAttemptsTotal" : { + "type" : "float" + }, + "TaskStartupFailurePercentage" : { + "type" : "float" + }, + "TaskStartupFailureTotal" : { + "type" : "float" + }, + "TaskStartupSuccessPercentage" : { + "type" : "float" + }, + "TaskStartupSuccessTotal" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_connect_connector_metric b/km-persistence/src/main/resources/es/template/ks_kafka_connect_connector_metric new file mode 100644 index 00000000..b26836a0 --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_connect_connector_metric @@ -0,0 +1,194 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_connect_connector_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "connectClusterId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "connectorName" : { + "type" : "keyword" + }, + "connectorNameAndClusterId" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "HealthState" : { + "type" : "float" + }, + "ConnectorTotalTaskCount" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ConnectorRunningTaskCount" : { + "type" : "float" + }, + "ConnectorPausedTaskCount" : { + "type" : "float" + }, + "ConnectorFailedTaskCount" : { + "type" : "float" + }, + "ConnectorUnassignedTaskCount" : { + "type" : "float" + }, + "BatchSizeAvg" : { + "type" : "float" + }, + "BatchSizeMax" : { + "type" : "float" + }, + "OffsetCommitAvgTimeMs" : { + "type" : "float" + }, + "OffsetCommitMaxTimeMs" : { + "type" : "float" + }, + "OffsetCommitFailurePercentage" : { + "type" : "float" + }, + "OffsetCommitSuccessPercentage" : { + "type" : "float" + }, + "PollBatchAvgTimeMs" : { + "type" : "float" + }, + "PollBatchMaxTimeMs" : { + "type" : "float" + }, + "SourceRecordActiveCount" : { + "type" : "float" + }, + "SourceRecordActiveCountAvg" : { + "type" : "float" + }, + "SourceRecordActiveCountMax" : { + "type" : "float" + }, + "SourceRecordPollRate" : { + "type" : "float" + }, + "SourceRecordPollTotal" : { + "type" : "float" + }, + "SourceRecordWriteRate" : { + "type" : "float" + }, + "SourceRecordWriteTotal" : { + "type" : "float" + }, + "OffsetCommitCompletionRate" : { + "type" : "float" + }, + "OffsetCommitCompletionTotal" : { + "type" : "float" + }, + "OffsetCommitSkipRate" : { + "type" : "float" + }, + "OffsetCommitSkipTotal" : { + "type" : "float" + }, + "PartitionCount" : { + "type" : "float" + }, + "PutBatchAvgTimeMs" : { + "type" : "float" + }, + "PutBatchMaxTimeMs" : { + "type" : "float" + }, + "SinkRecordActiveCount" : { + "type" : "float" + }, + "SinkRecordActiveCountAvg" : { + "type" : "float" + }, + "SinkRecordActiveCountMax" : { + "type" : "float" + }, + "SinkRecordLagMax" : { + "type" : "float" + }, + "SinkRecordReadRate" : { + "type" : "float" + }, + "SinkRecordReadTotal" : { + "type" : "float" + }, + "SinkRecordSendRate" : { + "type" : "float" + }, + "SinkRecordSendTotal" : { + "type" : "float" + }, + "DeadletterqueueProduceFailures" : { + "type" : "float" + }, + "DeadletterqueueProduceRequests" : { + "type" : "float" + }, + "LastErrorTimestamp" : { + "type" : "float" + }, + "TotalErrorsLogged" : { + "type" : "float" + }, + "TotalRecordErrors" : { + "type" : "float" + }, + "TotalRecordFailures" : { + "type" : "float" + }, + "TotalRecordsSkipped" : { + "type" : "float" + }, + "TotalRetries" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_group_metric b/km-persistence/src/main/resources/es/template/ks_kafka_group_metric new file mode 100644 index 00000000..3f04f16a --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_group_metric @@ -0,0 +1,74 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_group_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "group" : { + "type" : "keyword" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "HealthScore" : { + "type" : "float" + }, + "Lag" : { + "type" : "float" + }, + "OffsetConsumed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "groupMetric" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric b/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric new file mode 100644 index 00000000..68264660 --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric @@ -0,0 +1,65 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_replication_metric b/km-persistence/src/main/resources/es/template/ks_kafka_replication_metric new file mode 100644 index 00000000..a8ff4b53 --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_replication_metric @@ -0,0 +1,65 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_replication_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric b/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric new file mode 100644 index 00000000..b2eed35d --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric @@ -0,0 +1,116 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_topic_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "topic" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "BytesIn_min_15" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "BytesRejected" : { + "type" : "float" + }, + "PartitionURP" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ReplicationCount" : { + "type" : "float" + }, + "ReplicationBytesOut" : { + "type" : "float" + }, + "ReplicationBytesIn" : { + "type" : "float" + }, + "FailedFetchRequests" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "LogSize" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "FailedProduceRequests" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "brokerAgg" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + } \ No newline at end of file diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric b/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric new file mode 100644 index 00000000..e2efb41a --- /dev/null +++ b/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric @@ -0,0 +1,84 @@ +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_zookeeper_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "AvgRequestLatency" : { + "type" : "double" + }, + "MinRequestLatency" : { + "type" : "double" + }, + "MaxRequestLatency" : { + "type" : "double" + }, + "OutstandingRequests" : { + "type" : "double" + }, + "NodeCount" : { + "type" : "double" + }, + "WatchCount" : { + "type" : "double" + }, + "NumAliveConnections" : { + "type" : "double" + }, + "PacketsReceived" : { + "type" : "double" + }, + "PacketsSent" : { + "type" : "double" + }, + "EphemeralsCount" : { + "type" : "double" + }, + "ApproximateDataSize" : { + "type" : "double" + }, + "OpenFileDescriptorCount" : { + "type" : "double" + }, + "MaxFileDescriptorCount" : { + "type" : "double" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "type" : "date" + } + } + }, + "aliases" : { } + } \ No newline at end of file