调整v3.2版本容器化部署信息

km-dist/helm/Chart.yaml

@@ -1,16 +1,16 @@
 apiVersion: v2
 name: knowstreaming-manager
-description: knowstreaming-manager Helm chart
+description: knowstreaming-manager Helm chart 
 
 type: application
 
-version: 0.1.5
+version: 0.1.9
 
 maintainers:
 - email: didicloud@didiglobal.com
   name: didicloud
 
-appVersion: "3.0.0-beta.3"
+appVersion: "3.2.0"
 
 dependencies:
   - name: knowstreaming-web

km-dist/helm/charts/elasticsearch/values.yaml

@@ -175,6 +175,7 @@ antiAffinityTopologyKey: "kubernetes.io/hostname"
 # and that they will never end up on the same node. Setting this to soft will do this "best effort"
 antiAffinity: ""
 #antiAffinity: "hard"
+
 # This is the node affinity settings as defined in
 # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature
 nodeAffinity: {}

km-dist/helm/charts/knowstreaming-web/values.yaml

@@ -8,7 +8,7 @@ image:
   repository: knowstreaming/knowstreaming-ui
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "latest"
+  tag: "0.1.0"
 
 imagePullSecrets: []
 nameOverride: ""

km-dist/helm/charts/ksmysql/templates/statefulset.yaml

@@ -21,7 +21,7 @@ spec:
         {{- include "ksmysql.selectorLabels" . | nindent 8 }}
     spec:
       containers:
-        - image: knowstreaming/knowstreaming-mysql:latest
+        - image: knowstreaming/knowstreaming-mysql:0.7.0
           name: {{ .Chart.Name }}
           env:
             - name: MYSQL_DATABASE

km-dist/helm/templates/configmap.yaml

@@ -41,6 +41,7 @@ data:
           idle-timeout: 30000
           connection-test-query: SELECT 1
       logi-job:
+        enable: true 
         {{ if .Values.ksmysql.enabled }}
         jdbc-url: jdbc:mariadb://{{ .Values.ksmysql.service.name }}:{{ .Values.ksmysql.service.port }}/{{ .Values.ksmysql.mysql.dbname }}?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
         username: {{ .Values.ksmysql.mysql.username }}
@@ -77,6 +78,10 @@ data:
       config: classpath:logback-spring.xml
 
     thread-pool:
+      es:
+        search:                         # es查询线程池
+          thread-num: 20                # 线程池大小
+          queue-size: 10000             # 队列大小
       scheduled:
         thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
       collector:                      # 采集模块的配置
@@ -104,18 +109,25 @@ data:
         max-idle-client-num: 20       # 最大空闲客户端数
         max-total-client-num: 20      # 最大客户端数
         borrow-timeout-unit-ms: 5000  # 租借超时时间，单位秒
+      kafka-admin:
+        client-cnt: 1                 # 每个Kafka集群创建的KafkaAdminClient数
+
 
     es:
+      index:
+        expire: 15                    # 索引过期天数，15表示超过15天的索引会被KS过期删除
       client:
 {{ if .Values.elasticsearch.enabled }}
         address: elasticsearch-master:9200
 {{- else }}
         address: {{ .Values.elasticsearch.esClientAddress }}:{{ .Values.elasticsearch.esProt }}
+        pass: {{ .Values.elasticsearch.userPass }}
 {{- end }}
         client-cnt: 10
         io-thread-cnt: 2
         max-retry-cnt: 5
 
+
     # 普罗米修斯指标导出相关配置
     management:
       endpoints:
@@ -151,6 +163,7 @@ data:
     curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_group_metric -d '{"order":10,"index_patterns":["ks_kafka_group_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"group":{"type":"keyword"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"HealthScore":{"type":"float"},"Lag":{"type":"float"},"OffsetConsumed":{"type":"float"},"HealthCheckTotal":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"groupMetric":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
     curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_partition_metric -d '{"order":10,"index_patterns":["ks_kafka_partition_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"partitionId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"topic":{"type":"keyword"},"metrics":{"properties":{"LogStartOffset":{"type":"float"},"Messages":{"type":"float"},"LogEndOffset":{"type":"float"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
     curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{"order":10,"index_patterns":["ks_kafka_replication_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' && \
+    curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_zookeeper_metric -d '{"order":10,"index_patterns":["ks_kafka_zookeeper_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"AvgRequestLatency":{"type":"double"},"MinRequestLatency":{"type":"double"},"MaxRequestLatency":{"type":"double"},"OutstandingRequests":{"type":"double"},"NodeCount":{"type":"double"},"WatchCount":{"type":"double"},"NumAliveConnections":{"type":"double"},"PacketsReceived":{"type":"double"},"PacketsSent":{"type":"double"},"EphemeralsCount":{"type":"double"},"ApproximateDataSize":{"type":"double"},"OpenFileDescriptorCount":{"type":"double"},"MaxFileDescriptorCount":{"type":"double"}}},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","type":"date"}}},"aliases":{}}' && \
     curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{"order":10,"index_patterns":["ks_kafka_topic_metric*"],"settings":{"index":{"number_of_shards":"10"}},"mappings":{"properties":{"brokerId":{"type":"long"},"routingValue":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"topic":{"type":"keyword"},"clusterPhyId":{"type":"long"},"metrics":{"properties":{"BytesIn_min_15":{"type":"float"},"Messages":{"type":"float"},"BytesRejected":{"type":"float"},"PartitionURP":{"type":"float"},"HealthCheckTotal":{"type":"float"},"ReplicationCount":{"type":"float"},"ReplicationBytesOut":{"type":"float"},"ReplicationBytesIn":{"type":"float"},"FailedFetchRequests":{"type":"float"},"BytesIn_min_5":{"type":"float"},"HealthScore":{"type":"float"},"LogSize":{"type":"float"},"BytesOut":{"type":"float"},"BytesOut_min_15":{"type":"float"},"FailedProduceRequests":{"type":"float"},"BytesIn":{"type":"float"},"BytesOut_min_5":{"type":"float"},"MessagesIn":{"type":"float"},"TotalProduceRequests":{"type":"float"},"HealthCheckPassed":{"type":"float"}}},"brokerAgg":{"type":"keyword"},"key":{"type":"text","fields":{"keyword":{"ignore_above":256,"type":"keyword"}}},"timestamp":{"format":"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis","index":true,"type":"date","doc_values":true}}},"aliases":{}}' || \
     exit 1
     for i in {0..6};do
@@ -160,6 +173,8 @@ data:
         curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \
         curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \
         curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \
+        curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_zookeeper_metric${logdate} && \
         curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \
         exit 2
     done
+    

km-dist/helm/templates/deployment.yaml

@@ -33,6 +33,8 @@ spec:
       - name: init-config
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         command: ['/bin/bash', '/conf/init_es_index.sh']
+        resources:
+          {{- toYaml .Values.resources | nindent 10 }}
         volumeMounts:
           - name: configmap
             mountPath: /conf

km-dist/helm/values.yaml

@@ -3,7 +3,7 @@ replicaCount: 2
 image:
   repository: knowstreaming/knowstreaming-manager
   pullPolicy: IfNotPresent
-  tag: "latest"
+  tag: "0.7.0"
 
 imagePullSecrets: []
 nameOverride: ""
@@ -73,7 +73,7 @@ knowstreaming-web:
     image:
       repository: knowstreaming/knowstreaming-ui
       pullPolicy: IfNotPresent
-      tag: "latest"
+      tag: "0.7.0"
 
     service:
       type: NodePort
@@ -93,6 +93,7 @@ elasticsearch:
 #------------------------------------------------------------------
     esClientAddress: 10.96.64.13
     esProt: 8061
+    userPass: "username:password"   #ES账号密码，如果有账号密码，按照username:password 的格式填写，没有则不需要填写
 #------------------------------------------------------------------
     replicas: 3
     minimumMasterNodes: 2