mirror of
https://github.com/didi/KnowStreaming.git
synced 2026-01-08 15:52:15 +08:00
初始化3.0.0版本
This commit is contained in:
zengqiao
158
km-dist/helm/templates/configmap.yaml
Normal file
158
km-dist/helm/templates/configmap.yaml
Normal file
@@ -0,0 +1,158 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "knowstreaming-manager.fullname" . }}
|
||||
labels:
|
||||
app: {{ template "knowstreaming-manager.name" . }}
|
||||
chart: {{ template "knowstreaming-manager.chart" . }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
component: server
|
||||
data:
|
||||
application-test.yml: |
|
||||
server:
|
||||
port: 80 # 服务端口
|
||||
tomcat:
|
||||
accept-count: 1000
|
||||
max-connections: 10000
|
||||
|
||||
spring:
|
||||
application:
|
||||
name: know-streaming
|
||||
profiles:
|
||||
active: dev
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
jackson:
|
||||
time-zone: GMT+8
|
||||
datasource:
|
||||
know-streaming:
|
||||
{{ if .Values.ksmysql.enabled }}
|
||||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: {{ .Values.ksmysql.mysql.username }}
|
||||
password: {{ .Values.ksmysql.mysql.password }}
|
||||
{{- else }}
|
||||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: {{ .Values.ksmysql.username }}
|
||||
password: {{ .Values.ksmysql.password }}
|
||||
{{- end }}
|
||||
driver-class-name: org.mariadb.jdbc.Driver
|
||||
maximum-pool-size: 20
|
||||
idle-timeout: 30000
|
||||
connection-test-query: SELECT 1
|
||||
logi-job:
|
||||
{{ if .Values.ksmysql.enabled }}
|
||||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: {{ .Values.ksmysql.mysql.username }}
|
||||
password: {{ .Values.ksmysql.mysql.password }}
|
||||
{{- else }}
|
||||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: {{ .Values.ksmysql.username }}
|
||||
password: {{ .Values.ksmysql.password }}
|
||||
{{- end }}
|
||||
driver-class-name: org.mariadb.jdbc.Driver
|
||||
max-lifetime: 60000
|
||||
init-sql: true
|
||||
init-thread-num: 50
|
||||
max-thread-num: 100
|
||||
log-expire: 3 # 日志保存天数，以天为单位
|
||||
app-name: know-stream
|
||||
claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual
|
||||
logi-security:
|
||||
{{ if .Values.ksmysql.enabled }}
|
||||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: {{ .Values.ksmysql.mysql.username }}
|
||||
password: {{ .Values.ksmysql.mysql.password }}
|
||||
{{- else }}
|
||||
jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.mysqlAddress }}:{{ .Values.ksmysql.mysqlProt }}/{{ .Values.ksmysql.databasename }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
|
||||
username: {{ .Values.ksmysql.username }}
|
||||
password: {{ .Values.ksmysql.password }}
|
||||
{{- end }}
|
||||
driver-class-name: org.mariadb.jdbc.Driver
|
||||
app-name: know-streaming
|
||||
resource-extend-bean-name: myResourceExtendImpl
|
||||
|
||||
logging:
|
||||
config: classpath:logback-spring.xml
|
||||
|
||||
thread-pool:
|
||||
scheduled:
|
||||
thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
|
||||
collector:
|
||||
future-util:
|
||||
num: 1
|
||||
thread-num: 8
|
||||
queue-size: 10000
|
||||
select-suitable-enable: true
|
||||
suitable-queue-size: 1000
|
||||
task:
|
||||
heaven:
|
||||
thread-num: 20
|
||||
queue-size: 1000
|
||||
|
||||
|
||||
|
||||
client-pool:
|
||||
kafka-consumer:
|
||||
min-idle-client-num: 2 # 最小空闲客户端数
|
||||
max-idle-client-num: 20 # 最大空闲客户端数
|
||||
max-total-client-num: 20 # 最大客户端数
|
||||
borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒
|
||||
|
||||
{{ if .Values.elasticsearch.enabled }}
|
||||
es.client.address: elasticsearch-master:9200
|
||||
#es.client.address: {{ .Release.Name }}-elasticsearch:9200
|
||||
{{- else }}
|
||||
es.client.address: {{ .Values.elasticsearch.esClientAddress }}:{{ .Values.elasticsearch.esProt }}
|
||||
{{- end }}
|
||||
# es.client.pass: knowstreaming-manager
|
||||
# 集群自动均衡相关配置
|
||||
cluster-balance:
|
||||
ignored-topics:
|
||||
time-second: 300
|
||||
|
||||
# 普罗米修斯指标导出相关配置
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
base-path: /metrics
|
||||
exposure:
|
||||
include: '*'
|
||||
metrics:
|
||||
export:
|
||||
prometheus:
|
||||
descriptions: true
|
||||
enabled: true
|
||||
tags:
|
||||
application: know-streaming
|
||||
|
||||
|
||||
|
||||
init_es_index.sh: |
|
||||
#!/bin/bash
|
||||
{{ if .Values.elasticsearch.enabled }}
|
||||
esaddr=elasticsearch-master
|
||||
port=9200
|
||||
{{- else }}
|
||||
esaddr={{ .Values.elasticsearch.esClientAddress }}
|
||||
port={{ .Values.elasticsearch.esProt }}
|
||||
{{- end }}
|
||||
|
||||
curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_broker_metric -d '{"order":10,"index_patterns":["ks_kafka_broker_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"NetworkProcessorAvgIdle":{"type":"float"},"UnderReplicatedPartitions":{"type":"float"},"BytesIn_min_15":{"type":"float"},"HealthCheckTotal":{"type":"float"},"RequestHandlerAvgIdle":{"type":"float"},"connectionsCount":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"TotalRequestQueueSize":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"},"TotalResponseQueueSize":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_cluster_metric -d '{"order":10,"index_patterns":["ks_kafka_cluster_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"Connections":{"type":"double"},"BytesIn_min_15":{"type":"double"},"PartitionURP":{"type":"double"},"HealthScore_Topics":{"type":"double"},"EventQueueSize":{"type":"double"},"ActiveControllerCount":{"type":"double"},"GroupDeads":{"type":"double"},"BytesIn_min_5":{"type":"double"},"HealthCheckTotal_Topics":{"type":"double"},"Partitions":{"type":"double"},"BytesOut":{"type":"double"},"Groups":{"type":"double"},"BytesOut_min_15":{"type":"double"},"TotalRequestQueueSize":{"type":"double"},"HealthCheckPassed_Groups":{"type":"double"},"TotalProduceRequests":{"type":"double"},"HealthCheckPassed":{"type":"double"},"TotalLogSize":{"type":"double"},"GroupEmptys":{"type":"double"},"PartitionNoLeader":{"type":"double"},"HealthScore_Brokers":{"type":"double"},"Messages":{"type":"double"},"Topics":{"type":"double"},"PartitionMinISR_E":{"type":"double"},"HealthCheckTotal":{"type":"double"},"Brokers":{"type":"double"},"Replicas":{"type":"double"},"HealthCheckTotal_Groups":{"type":"double"},"GroupRebalances":{"type":"double"},"MessageIn":{"type":"double"},"HealthScore":{"type":"double"},"HealthCheckPassed_Topics":{"type":"double"},"HealthCheckTotal_Brokers":{"type":"double"},"PartitionMinISR_S":{"type":"double"},"BytesIn":{"type":"double"},"BytesOut_min_5":{"type":"double"},"GroupActives":{"type":"double"},"MessagesIn":{"type":"double"},"GroupReBalances":{"type":"double"},"HealthCheckPassed_Brokers":{"type":"double"},"HealthScore_Groups":{"type":"double"},"TotalResponseQueueSize":{"type":"double"},"Zookeepers":{"type":"double"},"LeaderMessages":{"type":"double"},"HealthScore_Cluster":{"type":"double"},"HealthCheckPassed_Cluster":{"type":"double"},"HealthCheckTotal_Cluster":{"type":"double"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","type":"date"}}},"aliases":{}}' && \
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_group_metric -d '{"order":10,"index_patterns":["ks_kafka_group_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"group":{"type":"keyword"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"HealthScore":{"type":"float"},"Lag":{"type":"float"},"OffsetConsumed":{"type":"float"},"HealthCheckTotal":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"groupMetric":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_partition_metric -d '{"order":10,"index_patterns":["ks_kafka_partition_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"LogStartOffset":{"type":"float"},"Messages":{"type":"float"},"LogEndOffset":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{"order":10,"index_patterns":["ks_kafka_replication_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
|
||||
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{"order":10,"index_patterns":["ks_kafka_topic_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"topic":{"type":"keyword"},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"BytesIn_min_15":{"type":"float"},"Messages":{"type":"float"},"BytesRejected":{"type":"float"},"PartitionURP":{"type":"float"},"HealthCheckTotal":{"type":"float"},"ReplicationCount":{"type":"float"},"ReplicationBytesOut":{"type":"float"},"ReplicationBytesIn":{"type":"float"},"FailedFetchRequests":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"LogSize":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"FailedProduceRequests":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"brokerAgg":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' || \
|
||||
exit 1
|
||||
for i in {0..6};do
|
||||
logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
|
||||
curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_broker_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_cluster_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \
|
||||
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \
|
||||
exit 2
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user