From ac7c32acd591c039d47650b8cf0878984d3e366c Mon Sep 17 00:00:00 2001 From: zengqiao Date: Mon, 9 Jan 2023 15:09:15 +0800 Subject: [PATCH] =?UTF-8?q?[Optimize]=E4=BC=98=E5=8C=96ES=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E5=8F=8A=E6=A8=A1=E7=89=88=E7=9A=84=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E6=96=87=E6=A1=A3(#832)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、订正不同地方索引模版的shard数存在不一致的问题； 2、删除多余的template.sh，统一使用init_es_template.sh； 3、init_es_template.sh中，增加connect相关索引模版的初始化脚本，删除replica 和 zookeper索引模版的初始化脚本； --- bin/init_es_template.sh | 363 ++++++-- docs/dev_guide/无数据排查文档.md | 7 +- km-dist/init/template/ks_kafka_group_metric | 2 +- .../init/template/ks_kafka_partition_metric | 2 +- km-dist/init/template/ks_kafka_topic_metric | 2 +- .../init/template/ks_kafka_zookeeper_metric | 2 +- km-dist/init/template/template.sh | 803 ------------------ .../es/template/ks_kafka_group_metric | 2 +- .../es/template/ks_kafka_partition_metric | 2 +- .../es/template/ks_kafka_topic_metric | 2 +- .../es/template/ks_kafka_zookeeper_metric | 2 +- 11 files changed, 301 insertions(+), 888 deletions(-) delete mode 100644 km-dist/init/template/template.sh diff --git a/bin/init_es_template.sh b/bin/init_es_template.sh index 86fcfb66..67e21685 100644 --- a/bin/init_es_template.sh +++ b/bin/init_es_template.sh @@ -13,7 +13,7 @@ curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' - ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : { @@ -115,7 +115,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : { @@ -302,7 +302,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { @@ -377,7 +377,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { @@ -436,72 +436,6 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl "aliases" : { } }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_replication_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{ "order" : 10, "index_patterns" : [ @@ -509,7 +443,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { @@ -626,7 +560,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : { @@ -704,6 +638,288 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl "aliases" : { } }' +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_connect_cluster_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_connect_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "connectClusterId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "ConnectorCount" : { + "type" : "float" + }, + "TaskCount" : { + "type" : "float" + }, + "ConnectorStartupAttemptsTotal" : { + "type" : "float" + }, + "ConnectorStartupFailurePercentage" : { + "type" : "float" + }, + "ConnectorStartupFailureTotal" : { + "type" : "float" + }, + "ConnectorStartupSuccessPercentage" : { + "type" : "float" + }, + "ConnectorStartupSuccessTotal" : { + "type" : "float" + }, + "TaskStartupAttemptsTotal" : { + "type" : "float" + }, + "TaskStartupFailurePercentage" : { + "type" : "float" + }, + "TaskStartupFailureTotal" : { + "type" : "float" + }, + "TaskStartupSuccessPercentage" : { + "type" : "float" + }, + "TaskStartupSuccessTotal" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_connect_connector_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_connect_connector_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "2" + } + }, + "mappings" : { + "properties" : { + "connectClusterId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "connectorName" : { + "type" : "keyword" + }, + "connectorNameAndClusterId" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "HealthState" : { + "type" : "float" + }, + "ConnectorTotalTaskCount" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ConnectorRunningTaskCount" : { + "type" : "float" + }, + "ConnectorPausedTaskCount" : { + "type" : "float" + }, + "ConnectorFailedTaskCount" : { + "type" : "float" + }, + "ConnectorUnassignedTaskCount" : { + "type" : "float" + }, + "BatchSizeAvg" : { + "type" : "float" + }, + "BatchSizeMax" : { + "type" : "float" + }, + "OffsetCommitAvgTimeMs" : { + "type" : "float" + }, + "OffsetCommitMaxTimeMs" : { + "type" : "float" + }, + "OffsetCommitFailurePercentage" : { + "type" : "float" + }, + "OffsetCommitSuccessPercentage" : { + "type" : "float" + }, + "PollBatchAvgTimeMs" : { + "type" : "float" + }, + "PollBatchMaxTimeMs" : { + "type" : "float" + }, + "SourceRecordActiveCount" : { + "type" : "float" + }, + "SourceRecordActiveCountAvg" : { + "type" : "float" + }, + "SourceRecordActiveCountMax" : { + "type" : "float" + }, + "SourceRecordPollRate" : { + "type" : "float" + }, + "SourceRecordPollTotal" : { + "type" : "float" + }, + "SourceRecordWriteRate" : { + "type" : "float" + }, + "SourceRecordWriteTotal" : { + "type" : "float" + }, + "OffsetCommitCompletionRate" : { + "type" : "float" + }, + "OffsetCommitCompletionTotal" : { + "type" : "float" + }, + "OffsetCommitSkipRate" : { + "type" : "float" + }, + "OffsetCommitSkipTotal" : { + "type" : "float" + }, + "PartitionCount" : { + "type" : "float" + }, + "PutBatchAvgTimeMs" : { + "type" : "float" + }, + "PutBatchMaxTimeMs" : { + "type" : "float" + }, + "SinkRecordActiveCount" : { + "type" : "float" + }, + "SinkRecordActiveCountAvg" : { + "type" : "float" + }, + "SinkRecordActiveCountMax" : { + "type" : "float" + }, + "SinkRecordLagMax" : { + "type" : "float" + }, + "SinkRecordReadRate" : { + "type" : "float" + }, + "SinkRecordReadTotal" : { + "type" : "float" + }, + "SinkRecordSendRate" : { + "type" : "float" + }, + "SinkRecordSendTotal" : { + "type" : "float" + }, + "DeadletterqueueProduceFailures" : { + "type" : "float" + }, + "DeadletterqueueProduceRequests" : { + "type" : "float" + }, + "LastErrorTimestamp" : { + "type" : "float" + }, + "TotalErrorsLogged" : { + "type" : "float" + }, + "TotalRecordErrors" : { + "type" : "float" + }, + "TotalRecordFailures" : { + "type" : "float" + }, + "TotalRecordsSkipped" : { + "type" : "float" + }, + "TotalRetries" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + for i in {0..6}; do logdate=_$(date -d "${i} day ago" +%Y-%m-%d) @@ -711,8 +927,9 @@ do curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_cluster_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_zookeeper_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_connect_cluster_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_connect_connector_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \ exit 2 done diff --git a/docs/dev_guide/无数据排查文档.md b/docs/dev_guide/无数据排查文档.md index 337c5b13..fd7886bc 100644 --- a/docs/dev_guide/无数据排查文档.md +++ b/docs/dev_guide/无数据排查文档.md @@ -216,7 +216,7 @@ curl http://{ES的IP地址}:{ES的端口号}/_cat/indices/ks_kafka* 查看KS索 #### 3.1.2、解决方案 -执行[/km-dist/init/template/template.sh](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/template/template.sh)脚本创建索引。 +执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来创建索引及模版。 @@ -245,8 +245,7 @@ curl -XDELETE {ES的IP地址}:{ES的端口号}/ks_kafka* curl -XDELETE {ES的IP地址}:{ES的端口号}/_template/ks_kafka* ``` -执行[/km-dist/init/template/template.sh](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/template/template.sh)脚本初始化索引和模板。 - +执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来创建索引及模版。 ### 3.3、原因三：集群Shard满 @@ -283,4 +282,4 @@ curl -XPUT -H"content-type:application/json" http://{ES的IP地址}:{ES的端 }' ``` -执行[/km-dist/init/template/template.sh](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/template/template.sh)脚本补全索引。 +执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来补全索引。 diff --git a/km-dist/init/template/ks_kafka_group_metric b/km-dist/init/template/ks_kafka_group_metric index 3f04f16a..24ff12de 100644 --- a/km-dist/init/template/ks_kafka_group_metric +++ b/km-dist/init/template/ks_kafka_group_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { diff --git a/km-dist/init/template/ks_kafka_partition_metric b/km-dist/init/template/ks_kafka_partition_metric index 68264660..51193c22 100644 --- a/km-dist/init/template/ks_kafka_partition_metric +++ b/km-dist/init/template/ks_kafka_partition_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { diff --git a/km-dist/init/template/ks_kafka_topic_metric b/km-dist/init/template/ks_kafka_topic_metric index b2eed35d..4a1aa70c 100644 --- a/km-dist/init/template/ks_kafka_topic_metric +++ b/km-dist/init/template/ks_kafka_topic_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { diff --git a/km-dist/init/template/ks_kafka_zookeeper_metric b/km-dist/init/template/ks_kafka_zookeeper_metric index e2efb41a..10c9324b 100644 --- a/km-dist/init/template/ks_kafka_zookeeper_metric +++ b/km-dist/init/template/ks_kafka_zookeeper_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : { diff --git a/km-dist/init/template/template.sh b/km-dist/init/template/template.sh deleted file mode 100644 index baae166b..00000000 --- a/km-dist/init/template/template.sh +++ /dev/null @@ -1,803 +0,0 @@ -esaddr=127.0.0.1 -port=8060 -curl -s --connect-timeout 10 -o /dev/null http://${esaddr}:${port}/_cat/nodes >/dev/null 2>&1 -if [ "$?" != "0" ];then - echo "Elasticserach 访问失败, 请安装完后检查并重新执行该脚本 " - exit -fi - -curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_broker_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_broker_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "NetworkProcessorAvgIdle" : { - "type" : "float" - }, - "UnderReplicatedPartitions" : { - "type" : "float" - }, - "BytesIn_min_15" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "RequestHandlerAvgIdle" : { - "type" : "float" - }, - "connectionsCount" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "TotalRequestQueueSize" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "TotalResponseQueueSize" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_cluster_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "Connections" : { - "type" : "double" - }, - "BytesIn_min_15" : { - "type" : "double" - }, - "PartitionURP" : { - "type" : "double" - }, - "HealthScore_Topics" : { - "type" : "double" - }, - "EventQueueSize" : { - "type" : "double" - }, - "ActiveControllerCount" : { - "type" : "double" - }, - "GroupDeads" : { - "type" : "double" - }, - "BytesIn_min_5" : { - "type" : "double" - }, - "HealthCheckTotal_Topics" : { - "type" : "double" - }, - "Partitions" : { - "type" : "double" - }, - "BytesOut" : { - "type" : "double" - }, - "Groups" : { - "type" : "double" - }, - "BytesOut_min_15" : { - "type" : "double" - }, - "TotalRequestQueueSize" : { - "type" : "double" - }, - "HealthCheckPassed_Groups" : { - "type" : "double" - }, - "TotalProduceRequests" : { - "type" : "double" - }, - "HealthCheckPassed" : { - "type" : "double" - }, - "TotalLogSize" : { - "type" : "double" - }, - "GroupEmptys" : { - "type" : "double" - }, - "PartitionNoLeader" : { - "type" : "double" - }, - "HealthScore_Brokers" : { - "type" : "double" - }, - "Messages" : { - "type" : "double" - }, - "Topics" : { - "type" : "double" - }, - "PartitionMinISR_E" : { - "type" : "double" - }, - "HealthCheckTotal" : { - "type" : "double" - }, - "Brokers" : { - "type" : "double" - }, - "Replicas" : { - "type" : "double" - }, - "HealthCheckTotal_Groups" : { - "type" : "double" - }, - "GroupRebalances" : { - "type" : "double" - }, - "MessageIn" : { - "type" : "double" - }, - "HealthScore" : { - "type" : "double" - }, - "HealthCheckPassed_Topics" : { - "type" : "double" - }, - "HealthCheckTotal_Brokers" : { - "type" : "double" - }, - "PartitionMinISR_S" : { - "type" : "double" - }, - "BytesIn" : { - "type" : "double" - }, - "BytesOut_min_5" : { - "type" : "double" - }, - "GroupActives" : { - "type" : "double" - }, - "MessagesIn" : { - "type" : "double" - }, - "GroupReBalances" : { - "type" : "double" - }, - "HealthCheckPassed_Brokers" : { - "type" : "double" - }, - "HealthScore_Groups" : { - "type" : "double" - }, - "TotalResponseQueueSize" : { - "type" : "double" - }, - "Zookeepers" : { - "type" : "double" - }, - "LeaderMessages" : { - "type" : "double" - }, - "HealthScore_Cluster" : { - "type" : "double" - }, - "HealthCheckPassed_Cluster" : { - "type" : "double" - }, - "HealthCheckTotal_Cluster" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_group_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_group_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "group" : { - "type" : "keyword" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "HealthScore" : { - "type" : "float" - }, - "Lag" : { - "type" : "float" - }, - "OffsetConsumed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "groupMetric" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_partition_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_replication_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_topic_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "topic" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "BytesIn_min_15" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "BytesRejected" : { - "type" : "float" - }, - "PartitionURP" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ReplicationCount" : { - "type" : "float" - }, - "ReplicationBytesOut" : { - "type" : "float" - }, - "ReplicationBytesIn" : { - "type" : "float" - }, - "FailedFetchRequests" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "LogSize" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "FailedProduceRequests" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "brokerAgg" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_zookeeper_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_zookeeper_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "AvgRequestLatency" : { - "type" : "double" - }, - "MinRequestLatency" : { - "type" : "double" - }, - "MaxRequestLatency" : { - "type" : "double" - }, - "OutstandingRequests" : { - "type" : "double" - }, - "NodeCount" : { - "type" : "double" - }, - "WatchCount" : { - "type" : "double" - }, - "NumAliveConnections" : { - "type" : "double" - }, - "PacketsReceived" : { - "type" : "double" - }, - "PacketsSent" : { - "type" : "double" - }, - "EphemeralsCount" : { - "type" : "double" - }, - "ApproximateDataSize" : { - "type" : "double" - }, - "OpenFileDescriptorCount" : { - "type" : "double" - }, - "MaxFileDescriptorCount" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_zookeeper_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_zookeeper_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "2" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "AvgRequestLatency" : { - "type" : "double" - }, - "MinRequestLatency" : { - "type" : "double" - }, - "MaxRequestLatency" : { - "type" : "double" - }, - "OutstandingRequests" : { - "type" : "double" - }, - "NodeCount" : { - "type" : "double" - }, - "WatchCount" : { - "type" : "double" - }, - "NumAliveConnections" : { - "type" : "double" - }, - "PacketsReceived" : { - "type" : "double" - }, - "PacketsSent" : { - "type" : "double" - }, - "EphemeralsCount" : { - "type" : "double" - }, - "ApproximateDataSize" : { - "type" : "double" - }, - "OpenFileDescriptorCount" : { - "type" : "double" - }, - "MaxFileDescriptorCount" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - }' - -for i in {0..6}; -do - logdate=_$(date -d "${i} day ago" +%Y-%m-%d) - curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_broker_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_cluster_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_zookeeper_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \ - exit 2 -done diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_group_metric b/km-persistence/src/main/resources/es/template/ks_kafka_group_metric index 3f04f16a..24ff12de 100644 --- a/km-persistence/src/main/resources/es/template/ks_kafka_group_metric +++ b/km-persistence/src/main/resources/es/template/ks_kafka_group_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric b/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric index 68264660..51193c22 100644 --- a/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric +++ b/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric b/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric index b2eed35d..4a1aa70c 100644 --- a/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric +++ b/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "6" } }, "mappings" : { diff --git a/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric b/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric index e2efb41a..10c9324b 100644 --- a/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric +++ b/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric @@ -5,7 +5,7 @@ ], "settings" : { "index" : { - "number_of_shards" : "10" + "number_of_shards" : "2" } }, "mappings" : {