mirror of
https://github.com/didi/KnowStreaming.git
synced 2025-12-24 20:22:12 +08:00
166 lines
16 KiB
YAML
166 lines
16 KiB
YAML
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: {{ include "knowstreaming-manager.fullname" . }}
|
||
labels:
|
||
app: {{ template "knowstreaming-manager.name" . }}
|
||
chart: {{ template "knowstreaming-manager.chart" . }}
|
||
release: {{ .Release.Name }}
|
||
heritage: {{ .Release.Service }}
|
||
component: server
|
||
data:
|
||
application-test.yml: |
|
||
server:
|
||
port: 80 # 服务端口
|
||
tomcat:
|
||
accept-count: 1000
|
||
max-connections: 10000
|
||
|
||
spring:
|
||
application:
|
||
name: know-streaming
|
||
profiles:
|
||
active: dev
|
||
main:
|
||
allow-bean-definition-overriding: true
|
||
jackson:
|
||
time-zone: GMT+8
|
||
datasource:
|
||
know-streaming:
|
||
{{ if .Values.ksmysql.enabled }}
|
||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||
username: {{ .Values.ksmysql.mysql.username }}
|
||
password: {{ .Values.ksmysql.mysql.password }}
|
||
{{- else }}
|
||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||
username: {{ .Values.ksmysql.username }}
|
||
password: {{ .Values.ksmysql.password }}
|
||
{{- end }}
|
||
driver-class-name: org.mariadb.jdbc.Driver
|
||
maximum-pool-size: 20
|
||
idle-timeout: 30000
|
||
connection-test-query: SELECT 1
|
||
logi-job:
|
||
{{ if .Values.ksmysql.enabled }}
|
||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||
username: {{ .Values.ksmysql.mysql.username }}
|
||
password: {{ .Values.ksmysql.mysql.password }}
|
||
{{- else }}
|
||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||
username: {{ .Values.ksmysql.username }}
|
||
password: {{ .Values.ksmysql.password }}
|
||
{{- end }}
|
||
driver-class-name: org.mariadb.jdbc.Driver
|
||
max-lifetime: 60000
|
||
init-sql: true
|
||
init-thread-num: 50
|
||
max-thread-num: 100
|
||
log-expire: 3 # 日志保存天数，以天为单位
|
||
app-name: know-stream
|
||
claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual
|
||
logi-security:
|
||
{{ if .Values.ksmysql.enabled }}
|
||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||
username: {{ .Values.ksmysql.mysql.username }}
|
||
password: {{ .Values.ksmysql.mysql.password }}
|
||
{{- else }}
|
||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||
username: {{ .Values.ksmysql.username }}
|
||
password: {{ .Values.ksmysql.password }}
|
||
{{- end }}
|
||
driver-class-name: org.mariadb.jdbc.Driver
|
||
app-name: know-streaming
|
||
resource-extend-bean-name: myResourceExtendImpl
|
||
login-extend-bean-name: logiSecurityDefaultLoginExtendImpl
|
||
|
||
logging:
|
||
config: classpath:logback-spring.xml
|
||
|
||
thread-pool:
|
||
scheduled:
|
||
thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
|
||
collector: # 采集模块的配置
|
||
future-util: # 采集模块线程池配置
|
||
num: 3 # 线程池个数
|
||
thread-num: 64 # 每个线程池核心线程数
|
||
queue-size: 10000 # 每个线程池队列大小
|
||
select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改
|
||
suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改
|
||
task: # 任务模块的配置
|
||
metrics: # metrics采集任务配置
|
||
thread-num: 18 # metrics采集任务线程池核心线程数
|
||
queue-size: 180 # metrics采集任务线程池队列大小
|
||
metadata: # metadata同步任务配置
|
||
thread-num: 27 # metadata同步任务线程池核心线程数
|
||
queue-size: 270 # metadata同步任务线程池队列大小
|
||
common: # 剩余其他任务配置
|
||
thread-num: 15 # 剩余其他任务线程池核心线程数
|
||
queue-size: 150 # 剩余其他任务线程池队列大小
|
||
|
||
|
||
client-pool:
|
||
kafka-consumer:
|
||
min-idle-client-num: 2 # 最小空闲客户端数
|
||
max-idle-client-num: 20 # 最大空闲客户端数
|
||
max-total-client-num: 20 # 最大客户端数
|
||
borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒
|
||
|
||
es:
|
||
client:
|
||
{{ if .Values.elasticsearch.enabled }}
|
||
address: elasticsearch-master:9200
|
||
{{- else }}
|
||
address: {{ .Values.elasticsearch.esClientAddress }}:{{ .Values.elasticsearch.esProt }}
|
||
{{- end }}
|
||
client-cnt: 10
|
||
io-thread-cnt: 2
|
||
max-retry-cnt: 5
|
||
|
||
# 普罗米修斯指标导出相关配置
|
||
management:
|
||
endpoints:
|
||
web:
|
||
base-path: /metrics
|
||
exposure:
|
||
include: '*'
|
||
health:
|
||
elasticsearch:
|
||
enabled: false
|
||
metrics:
|
||
export:
|
||
prometheus:
|
||
descriptions: true
|
||
enabled: true
|
||
tags:
|
||
application: know-streaming
|
||
|
||
|
||
|
||
init_es_index.sh: |
|
||
#!/bin/bash
|
||
{{ if .Values.elasticsearch.enabled }}
|
||
esaddr=elasticsearch-master
|
||
port=9200
|
||
{{- else }}
|
||
esaddr={{ .Values.elasticsearch.esClientAddress }}
|
||
port={{ .Values.elasticsearch.esProt }}
|
||
{{- end }}
|
||
|
||
curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_broker_metric -d '{"order":10,"index_patterns":["ks_kafka_broker_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"NetworkProcessorAvgIdle":{"type":"float"},"UnderReplicatedPartitions":{"type":"float"},"BytesIn_min_15":{"type":"float"},"HealthCheckTotal":{"type":"float"},"RequestHandlerAvgIdle":{"type":"float"},"connectionsCount":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"TotalRequestQueueSize":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"},"TotalResponseQueueSize":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_cluster_metric -d '{"order":10,"index_patterns":["ks_kafka_cluster_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"Connections":{"type":"double"},"BytesIn_min_15":{"type":"double"},"PartitionURP":{"type":"double"},"HealthScore_Topics":{"type":"double"},"EventQueueSize":{"type":"double"},"ActiveControllerCount":{"type":"double"},"GroupDeads":{"type":"double"},"BytesIn_min_5":{"type":"double"},"HealthCheckTotal_Topics":{"type":"double"},"Partitions":{"type":"double"},"BytesOut":{"type":"double"},"Groups":{"type":"double"},"BytesOut_min_15":{"type":"double"},"TotalRequestQueueSize":{"type":"double"},"HealthCheckPassed_Groups":{"type":"double"},"TotalProduceRequests":{"type":"double"},"HealthCheckPassed":{"type":"double"},"TotalLogSize":{"type":"double"},"GroupEmptys":{"type":"double"},"PartitionNoLeader":{"type":"double"},"HealthScore_Brokers":{"type":"double"},"Messages":{"type":"double"},"Topics":{"type":"double"},"PartitionMinISR_E":{"type":"double"},"HealthCheckTotal":{"type":"double"},"Brokers":{"type":"double"},"Replicas":{"type":"double"},"HealthCheckTotal_Groups":{"type":"double"},"GroupRebalances":{"type":"double"},"MessageIn":{"type":"double"},"HealthScore":{"type":"double"},"HealthCheckPassed_Topics":{"type":"double"},"HealthCheckTotal_Brokers":{"type":"double"},"PartitionMinISR_S":{"type":"double"},"BytesIn":{"type":"double"},"BytesOut_min_5":{"type":"double"},"GroupActives":{"type":"double"},"MessagesIn":{"type":"double"},"GroupReBalances":{"type":"double"},"HealthCheckPassed_Brokers":{"type":"double"},"HealthScore_Groups":{"type":"double"},"TotalResponseQueueSize":{"type":"double"},"Zookeepers":{"type":"double"},"LeaderMessages":{"type":"double"},"HealthScore_Cluster":{"type":"double"},"HealthCheckPassed_Cluster":{"type":"double"},"HealthCheckTotal_Cluster":{"type":"double"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","type":"date"}}},"aliases":{}}' && \
|
||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_group_metric -d '{"order":10,"index_patterns":["ks_kafka_group_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"group":{"type":"keyword"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"HealthScore":{"type":"float"},"Lag":{"type":"float"},"OffsetConsumed":{"type":"float"},"HealthCheckTotal":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"groupMetric":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_partition_metric -d '{"order":10,"index_patterns":["ks_kafka_partition_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"LogStartOffset":{"type":"float"},"Messages":{"type":"float"},"LogEndOffset":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{"order":10,"index_patterns":["ks_kafka_replication_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{"order":10,"index_patterns":["ks_kafka_topic_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"topic":{"type":"keyword"},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"BytesIn_min_15":{"type":"float"},"Messages":{"type":"float"},"BytesRejected":{"type":"float"},"PartitionURP":{"type":"float"},"HealthCheckTotal":{"type":"float"},"ReplicationCount":{"type":"float"},"ReplicationBytesOut":{"type":"float"},"ReplicationBytesIn":{"type":"float"},"FailedFetchRequests":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"LogSize":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"FailedProduceRequests":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"brokerAgg":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' || \
|
||
exit 1
|
||
for i in {0..6};do
|
||
logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
|
||
curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_broker_metric${logdate} && \
|
||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_cluster_metric${logdate} && \
|
||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \
|
||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \
|
||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \
|
||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \
|
||
exit 2
|
||
done
|