apiVersion: v1 kind: ConfigMap metadata: name: {{ include "knowstreaming-manager.fullname" . }} labels: app: {{ template "knowstreaming-manager.name" . }} chart: {{ template "knowstreaming-manager.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} component: server data: application-test.yml: | server: port: 80 # 服务端口 tomcat: accept-count: 1000 max-connections: 10000 spring: application: name: know-streaming profiles: active: dev main: allow-bean-definition-overriding: true jackson: time-zone: GMT+8 datasource: know-streaming: {{ if .Values.ksmysql.enabled }} jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true username: {{ .Values.ksmysql.mysql.username }} password: {{ .Values.ksmysql.mysql.password }} {{- else }} jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true username: {{ .Values.ksmysql.username }} password: {{ .Values.ksmysql.password }} {{- end }} driver-class-name: org.mariadb.jdbc.Driver maximum-pool-size: 20 idle-timeout: 30000 connection-test-query: SELECT 1 logi-job: enable: true {{ if .Values.ksmysql.enabled }} jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true username: {{ .Values.ksmysql.mysql.username }} password: {{ .Values.ksmysql.mysql.password }} {{- else }} jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true username: {{ .Values.ksmysql.username }} password: {{ .Values.ksmysql.password }} {{- end }} driver-class-name: org.mariadb.jdbc.Driver max-lifetime: 60000 init-sql: true init-thread-num: 50 max-thread-num: 100 log-expire: 3 # 日志保存天数，以天为单位 app-name: know-stream claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual logi-security: {{ if .Values.ksmysql.enabled }} jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true username: {{ .Values.ksmysql.mysql.username }} password: {{ .Values.ksmysql.mysql.password }} {{- else }} jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true username: {{ .Values.ksmysql.username }} password: {{ .Values.ksmysql.password }} {{- end }} driver-class-name: org.mariadb.jdbc.Driver app-name: know-streaming resource-extend-bean-name: myResourceExtendImpl login-extend-bean-name: logiSecurityDefaultLoginExtendImpl logging: config: classpath:logback-spring.xml thread-pool: es: search: # es查询线程池 thread-num: 20 # 线程池大小 queue-size: 10000 # 队列大小 scheduled: thread-num: 2 # @Scheduled任务的线程池大小，默认是一个 collector: # 采集模块的配置 future-util: # 采集模块线程池配置 num: 3 # 线程池个数 thread-num: 64 # 每个线程池核心线程数 queue-size: 10000 # 每个线程池队列大小 select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改 suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改 task: # 任务模块的配置 metrics: # metrics采集任务配置 thread-num: 18 # metrics采集任务线程池核心线程数 queue-size: 180 # metrics采集任务线程池队列大小 metadata: # metadata同步任务配置 thread-num: 27 # metadata同步任务线程池核心线程数 queue-size: 270 # metadata同步任务线程池队列大小 common: # 剩余其他任务配置 thread-num: 15 # 剩余其他任务线程池核心线程数 queue-size: 150 # 剩余其他任务线程池队列大小 client-pool: kafka-consumer: min-idle-client-num: 2 # 最小空闲客户端数 max-idle-client-num: 20 # 最大空闲客户端数 max-total-client-num: 20 # 最大客户端数 borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒 kafka-admin: client-cnt: 1 # 每个Kafka集群创建的KafkaAdminClient数 es: index: expire: 15 # 索引过期天数，15表示超过15天的索引会被KS过期删除 client: {{ if .Values.elasticsearch.enabled }} address: elasticsearch-master:9200 {{- else }} address: {{ .Values.elasticsearch.esClientAddress }}:{{ .Values.elasticsearch.esProt }} pass: {{ .Values.elasticsearch.userPass }} {{- end }} client-cnt: 10 io-thread-cnt: 2 max-retry-cnt: 5 # 普罗米修斯指标导出相关配置 management: endpoints: web: base-path: /metrics exposure: include: '*' health: elasticsearch: enabled: false metrics: export: prometheus: descriptions: true enabled: true tags: application: know-streaming init_es_index.sh: | #!/bin/bash {{ if .Values.elasticsearch.enabled }} esaddr=elasticsearch-master port=9200 {{- else }} esaddr={{ .Values.elasticsearch.esClientAddress }} port={{ .Values.elasticsearch.esProt }} {{- end }} curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_broker_metric -d '{"order":10,"index_patterns":["ks_kafka_broker_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"NetworkProcessorAvgIdle":{"type":"float"},"UnderReplicatedPartitions":{"type":"float"},"BytesIn_min_15":{"type":"float"},"HealthCheckTotal":{"type":"float"},"RequestHandlerAvgIdle":{"type":"float"},"connectionsCount":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"TotalRequestQueueSize":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"},"TotalResponseQueueSize":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_cluster_metric -d '{"order":10,"index_patterns":["ks_kafka_cluster_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"Connections":{"type":"double"},"BytesIn_min_15":{"type":"double"},"PartitionURP":{"type":"double"},"HealthScore_Topics":{"type":"double"},"EventQueueSize":{"type":"double"},"ActiveControllerCount":{"type":"double"},"GroupDeads":{"type":"double"},"BytesIn_min_5":{"type":"double"},"HealthCheckTotal_Topics":{"type":"double"},"Partitions":{"type":"double"},"BytesOut":{"type":"double"},"Groups":{"type":"double"},"BytesOut_min_15":{"type":"double"},"TotalRequestQueueSize":{"type":"double"},"HealthCheckPassed_Groups":{"type":"double"},"TotalProduceRequests":{"type":"double"},"HealthCheckPassed":{"type":"double"},"TotalLogSize":{"type":"double"},"GroupEmptys":{"type":"double"},"PartitionNoLeader":{"type":"double"},"HealthScore_Brokers":{"type":"double"},"Messages":{"type":"double"},"Topics":{"type":"double"},"PartitionMinISR_E":{"type":"double"},"HealthCheckTotal":{"type":"double"},"Brokers":{"type":"double"},"Replicas":{"type":"double"},"HealthCheckTotal_Groups":{"type":"double"},"GroupRebalances":{"type":"double"},"MessageIn":{"type":"double"},"HealthScore":{"type":"double"},"HealthCheckPassed_Topics":{"type":"double"},"HealthCheckTotal_Brokers":{"type":"double"},"PartitionMinISR_S":{"type":"double"},"BytesIn":{"type":"double"},"BytesOut_min_5":{"type":"double"},"GroupActives":{"type":"double"},"MessagesIn":{"type":"double"},"GroupReBalances":{"type":"double"},"HealthCheckPassed_Brokers":{"type":"double"},"HealthScore_Groups":{"type":"double"},"TotalResponseQueueSize":{"type":"double"},"Zookeepers":{"type":"double"},"LeaderMessages":{"type":"double"},"HealthScore_Cluster":{"type":"double"},"HealthCheckPassed_Cluster":{"type":"double"},"HealthCheckTotal_Cluster":{"type":"double"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","type":"date"}}},"aliases":{}}' && \ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_group_metric -d '{"order":10,"index_patterns":["ks_kafka_group_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"group":{"type":"keyword"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"HealthScore":{"type":"float"},"Lag":{"type":"float"},"OffsetConsumed":{"type":"float"},"HealthCheckTotal":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"groupMetric":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_partition_metric -d '{"order":10,"index_patterns":["ks_kafka_partition_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"LogStartOffset":{"type":"float"},"Messages":{"type":"float"},"LogEndOffset":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{"order":10,"index_patterns":["ks_kafka_replication_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_zookeeper_metric -d '{"order":10,"index_patterns":["ks_kafka_zookeeper_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"AvgRequestLatency":{"type":"double"},"MinRequestLatency":{"type":"double"},"MaxRequestLatency":{"type":"double"},"OutstandingRequests":{"type":"double"},"NodeCount":{"type":"double"},"WatchCount":{"type":"double"},"NumAliveConnections":{"type":"double"},"PacketsReceived":{"type":"double"},"PacketsSent":{"type":"double"},"EphemeralsCount":{"type":"double"},"ApproximateDataSize":{"type":"double"},"OpenFileDescriptorCount":{"type":"double"},"MaxFileDescriptorCount":{"type":"double"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","type":"date"}}},"aliases":{}}' && \ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{"order":10,"index_patterns":["ks_kafka_topic_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"topic":{"type":"keyword"},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"BytesIn_min_15":{"type":"float"},"Messages":{"type":"float"},"BytesRejected":{"type":"float"},"PartitionURP":{"type":"float"},"HealthCheckTotal":{"type":"float"},"ReplicationCount":{"type":"float"},"ReplicationBytesOut":{"type":"float"},"ReplicationBytesIn":{"type":"float"},"FailedFetchRequests":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"LogSize":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"FailedProduceRequests":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"brokerAgg":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' || \ exit 1 for i in {0..6};do logdate=_$(date -d "${i} day ago" +%Y-%m-%d) curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_broker_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_cluster_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_zookeeper_metric${logdate} && \ curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \ exit 2 done