From 26916f6632c7ccc37722958f98c12584b4a7d619 Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Wed, 21 Sep 2022 12:55:43 +0800 Subject: [PATCH 01/23] =?UTF-8?q?1.=20[=E5=8D=95=E6=9C=BA=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=89=8B=E5=86=8C.md]docker-compose=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0=202.=20=E6=9B=B4=E6=94=B9docker-compose=E4=B8=ADui?= =?UTF-8?q?=E5=AF=B9=E5=A4=96=E8=AE=BF=E9=97=AEport=E4=B8=BA80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 57 ++++++++++++++++++------------ 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index b429b9d5..0d55c0cb 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -92,9 +92,10 @@ helm pull knowstreaming/knowstreaming-manager #### 2.1.3.2、Docker Compose ```yml version: "3" - +# https://hub.docker.com/u/knowstreaming 在此处寻找最新镜像版本 +# mysql与es可以使用自己搭建的服务,调整对应配置即可 services: - + # *不要调整knowstreaming-manager服务名称，ui中会用到 knowstreaming-manager: image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming-manager @@ -110,33 +111,36 @@ services: - /ks-start.sh environment: TZ: Asia/Shanghai - + # mysql服务地址 SERVER_MYSQL_ADDRESS: knowstreaming-mysql:3306 + # mysql数据库名 SERVER_MYSQL_DB: know_streaming + # mysql用户名 SERVER_MYSQL_USER: root + # mysql用户密码 SERVER_MYSQL_PASSWORD: admin2022_ - + # es服务地址 SERVER_ES_ADDRESS: elasticsearch-single:9200 - + # 服务JVM参数 JAVA_OPTS: -Xmx1g -Xms1g - -# extra_hosts: -# - "hostname:x.x.x.x" -# volumes: -# - /ks/manage/log:/logs + # 对于kafka中ADVERTISED_LISTENERS填写的hostname可以通过该方式完成 + # extra_hosts: + # - "hostname:x.x.x.x" + # 服务日志路径 + # volumes: + # - /ks/manage/log:/logs knowstreaming-ui: image: knowstreaming/knowstreaming-ui:0.2.0-test1 container_name: knowstreaming-ui restart: always ports: - - '18092:80' + - '80:80' environment: TZ: Asia/Shanghai depends_on: - knowstreaming-manager -# extra_hosts: -# - "hostname:x.x.x.x" - + # extra_hosts: + # - "hostname:x.x.x.x" elasticsearch-single: image: docker.io/library/elasticsearch:7.6.2 container_name: elasticsearch-single @@ -144,16 +148,21 @@ services: expose: - 9200 - 9300 -# ports: -# - '9200:9200' -# - '9300:9300' + # ports: + # - '9200:9200' + # - '9300:9300' environment: TZ: Asia/Shanghai + # es的JVM参数 ES_JAVA_OPTS: -Xms512m -Xmx512m + # 单节点配置，多节点集群参考 https://www.elastic.co/guide/en/elasticsearch/reference/7.6/docker.html#docker-compose-file discovery.type: single-node -# volumes: -# - /ks/es/data:/usr/share/elasticsearch/data + # 数据持久化路径 + # volumes: + # - /ks/es/data:/usr/share/elasticsearch/data + # es初始化服务，与manager使用同一镜像 + # 首次启动es需初始化模版和索引,后续会自动创建 knowstreaming-init: image: knowstreaming/knowstreaming-manager:0.2.0-test container_name: knowstreaming_init @@ -164,22 +173,26 @@ services: - /es_template_create.sh environment: TZ: Asia/Shanghai + # es服务地址 SERVER_ES_ADDRESS: elasticsearch-single:9200 - knowstreaming-mysql: image: knowstreaming/knowstreaming-mysql:0.2.0-test container_name: knowstreaming-mysql restart: always environment: TZ: Asia/Shanghai + # root 用户密码 MYSQL_ROOT_PASSWORD: admin2022_ + # 初始化时创建的数据库名称 MYSQL_DATABASE: know_streaming + # 通配所有host,可以访问远程 MYSQL_ROOT_HOST: '%' expose: - 3306 -# ports: -# - '3306:3306' + # ports: + # - '3306:3306' + # 数据持久化路径 # volumes: # - /ks/mysql/data:/data/mysql ``` From a2f8b1a851ac18a49256807cad89b02a8b1ea93d Mon Sep 17 00:00:00 2001 From: haoqi <1148648445@qq.com> Date: Thu, 22 Sep 2022 19:46:21 +0800 Subject: [PATCH 02/23] =?UTF-8?q?1.=20[=E5=8D=95=E6=9C=BA=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=89=8B=E5=86=8C.md]docker-compose=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 72 +++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index 0d55c0cb..bc895e10 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -90,10 +90,50 @@ helm pull knowstreaming/knowstreaming-manager   #### 2.1.3.2、Docker Compose -```yml -version: "3" +**环境依赖** + +- [Docker](https://docs.docker.com/engine/install/) +- [Docker Compose](https://docs.docker.com/compose/install/) + + +**安装命令** +```bash +# `v3.0.0-beta.2`版本开始，默认账号密码为`admin` / `admin`； # https://hub.docker.com/u/knowstreaming 在此处寻找最新镜像版本 # mysql与es可以使用自己搭建的服务,调整对应配置即可 + +# 复制docker-compose.yml到指定位置后执行下方命令即可启动 +docker-compose up -d +``` + +**验证安装** +```shell +docker-compose ps +# 验证启动 - 状态为 UP 则表示成功 + Name Command State Ports +---------------------------------------------------------------------------------------------------- +elasticsearch-single /usr/local/bin/docker-entr ... Up 9200/tcp, 9300/tcp +knowstreaming-init /bin/bash /es_template_cre ... Up +knowstreaming-manager /bin/sh /ks-start.sh Up 80/tcp +knowstreaming-mysql /entrypoint.sh mysqld Up (health: starting) 3306/tcp, 33060/tcp +knowstreaming-ui /docker-entrypoint.sh ngin ... Up 0.0.0.0:80->80/tcp + +# 稍等一分钟左右 knowstreaming-init 会退出，表示es初始化完成，可以访问页面 + Name Command State Ports +------------------------------------------------------------------------------------------- +knowstreaming-init /bin/bash /es_template_cre ... Exit 0 +knowstreaming-mysql /entrypoint.sh mysqld Up (healthy) 3306/tcp, 33060/tcp +``` + +**访问** +```http request +http://127.0.0.1:80/ +``` + + +**docker-compose.yml** +```yml +version: "2" services: # *不要调整knowstreaming-manager服务名称，ui中会用到 knowstreaming-manager: @@ -124,11 +164,11 @@ services: # 服务JVM参数 JAVA_OPTS: -Xmx1g -Xms1g # 对于kafka中ADVERTISED_LISTENERS填写的hostname可以通过该方式完成 - # extra_hosts: - # - "hostname:x.x.x.x" +# extra_hosts: +# - "hostname:x.x.x.x" # 服务日志路径 - # volumes: - # - /ks/manage/log:/logs +# volumes: +# - /ks/manage/log:/logs knowstreaming-ui: image: knowstreaming/knowstreaming-ui:0.2.0-test1 container_name: knowstreaming-ui @@ -139,8 +179,8 @@ services: TZ: Asia/Shanghai depends_on: - knowstreaming-manager - # extra_hosts: - # - "hostname:x.x.x.x" +# extra_hosts: +# - "hostname:x.x.x.x" elasticsearch-single: image: docker.io/library/elasticsearch:7.6.2 container_name: elasticsearch-single @@ -148,9 +188,9 @@ services: expose: - 9200 - 9300 - # ports: - # - '9200:9200' - # - '9300:9300' +# ports: +# - '9200:9200' +# - '9300:9300' environment: TZ: Asia/Shanghai # es的JVM参数 @@ -158,14 +198,14 @@ services: # 单节点配置，多节点集群参考 https://www.elastic.co/guide/en/elasticsearch/reference/7.6/docker.html#docker-compose-file discovery.type: single-node # 数据持久化路径 - # volumes: - # - /ks/es/data:/usr/share/elasticsearch/data +# volumes: +# - /ks/es/data:/usr/share/elasticsearch/data # es初始化服务，与manager使用同一镜像 # 首次启动es需初始化模版和索引,后续会自动创建 knowstreaming-init: image: knowstreaming/knowstreaming-manager:0.2.0-test - container_name: knowstreaming_init + container_name: knowstreaming-init depends_on: - elasticsearch-single command: @@ -190,8 +230,8 @@ services: MYSQL_ROOT_HOST: '%' expose: - 3306 - # ports: - # - '3306:3306' +# ports: +# - '3306:3306' # 数据持久化路径 # volumes: # - /ks/mysql/data:/data/mysql From eac76d7ad0b25738bf7df623e6f652bae407bcc2 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 11:36:01 +0800 Subject: [PATCH 03/23] helm update 3.0.0-beta.3 --- km-dist/helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/km-dist/helm/values.yaml b/km-dist/helm/values.yaml index 2c422f61..293bd001 100644 --- a/km-dist/helm/values.yaml +++ b/km-dist/helm/values.yaml @@ -3,7 +3,7 @@ replicaCount: 2 image: repository: knowstreaming/knowstreaming-manager pullPolicy: IfNotPresent - tag: "0.2.0" + tag: "latest" imagePullSecrets: [] nameOverride: "" @@ -73,7 +73,7 @@ knowstreaming-web: image: repository: knowstreaming/knowstreaming-ui pullPolicy: IfNotPresent - tag: "0.2.0" + tag: "latest" service: type: NodePort From 1aa89527a647ac0711fe8fee0589f39ceef68523 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 11:36:46 +0800 Subject: [PATCH 04/23] helm update 3.0.0-beta.3 --- km-dist/helm/charts/ksmysql/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/km-dist/helm/charts/ksmysql/templates/statefulset.yaml b/km-dist/helm/charts/ksmysql/templates/statefulset.yaml index a536f2f8..d61c506a 100644 --- a/km-dist/helm/charts/ksmysql/templates/statefulset.yaml +++ b/km-dist/helm/charts/ksmysql/templates/statefulset.yaml @@ -21,7 +21,7 @@ spec: {{- include "ksmysql.selectorLabels" . | nindent 8 }} spec: containers: - - image: knowstreaming/knowstreaming-mysql:0.2.0 + - image: knowstreaming/knowstreaming-mysql:latest name: {{ .Chart.Name }} env: - name: MYSQL_DATABASE From 26b1846bb4142f2fd790863e957256c944450b09 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:03:14 +0800 Subject: [PATCH 05/23] Create docker-compose.yml --- km-dist/docker/docker-compose.yml | 102 ++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 km-dist/docker/docker-compose.yml diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml new file mode 100644 index 00000000..f546db77 --- /dev/null +++ b/km-dist/docker/docker-compose.yml @@ -0,0 +1,102 @@ +version: "2" +services: + # *不要调整knowstreaming-manager服务名称，ui中会用到 + knowstreaming-manager: + image: knowstreaming/knowstreaming-manager:0.2.0-test + container_name: knowstreaming-manager + privileged: true + restart: always + depends_on: + - elasticsearch-single + - knowstreaming-mysql + expose: + - 80 + command: + - /bin/sh + - /ks-start.sh + environment: + TZ: Asia/Shanghai + # mysql服务地址 + SERVER_MYSQL_ADDRESS: knowstreaming-mysql:3306 + # mysql数据库名 + SERVER_MYSQL_DB: know_streaming + # mysql用户名 + SERVER_MYSQL_USER: root + # mysql用户密码 + SERVER_MYSQL_PASSWORD: admin2022_ + # es服务地址 + SERVER_ES_ADDRESS: elasticsearch-single:9200 + # 服务JVM参数 + JAVA_OPTS: -Xmx1g -Xms1g + # 对于kafka中ADVERTISED_LISTENERS填写的hostname可以通过该方式完成 +# extra_hosts: +# - "hostname:x.x.x.x" + # 服务日志路径 +# volumes: +# - /ks/manage/log:/logs + knowstreaming-ui: + image: knowstreaming/knowstreaming-ui:0.2.0-test1 + container_name: knowstreaming-ui + restart: always + ports: + - '80:80' + environment: + TZ: Asia/Shanghai + depends_on: + - knowstreaming-manager +# extra_hosts: +# - "hostname:x.x.x.x" + elasticsearch-single: + image: docker.io/library/elasticsearch:7.6.2 + container_name: elasticsearch-single + restart: always + expose: + - 9200 + - 9300 +# ports: +# - '9200:9200' +# - '9300:9300' + environment: + TZ: Asia/Shanghai + # es的JVM参数 + ES_JAVA_OPTS: -Xms512m -Xmx512m + # 单节点配置，多节点集群参考 https://www.elastic.co/guide/en/elasticsearch/reference/7.6/docker.html#docker-compose-file + discovery.type: single-node + # 数据持久化路径 +# volumes: +# - /ks/es/data:/usr/share/elasticsearch/data + + # es初始化服务，与manager使用同一镜像 + # 首次启动es需初始化模版和索引,后续会自动创建 + knowstreaming-init: + image: knowstreaming/knowstreaming-manager:0.2.0-test + container_name: knowstreaming-init + depends_on: + - elasticsearch-single + command: + - /bin/bash + - /es_template_create.sh + environment: + TZ: Asia/Shanghai + # es服务地址 + SERVER_ES_ADDRESS: elasticsearch-single:9200 + + knowstreaming-mysql: + image: knowstreaming/knowstreaming-mysql:0.2.0-test + container_name: knowstreaming-mysql + restart: always + environment: + TZ: Asia/Shanghai + # root 用户密码 + MYSQL_ROOT_PASSWORD: admin2022_ + # 初始化时创建的数据库名称 + MYSQL_DATABASE: know_streaming + # 通配所有host,可以访问远程 + MYSQL_ROOT_HOST: '%' + expose: + - 3306 +# ports: +# - '3306:3306' + # 数据持久化路径 +# volumes: +# - /ks/mysql/data:/data/mysql From a4186085d3b996daf35a866ddeaf9f33ef800c32 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:05:05 +0800 Subject: [PATCH 06/23] Create es_template_create.sh --- km-dist/docker/es_template_create.sh | 661 +++++++++++++++++++++++++++ 1 file changed, 661 insertions(+) create mode 100644 km-dist/docker/es_template_create.sh diff --git a/km-dist/docker/es_template_create.sh b/km-dist/docker/es_template_create.sh new file mode 100644 index 00000000..615bf54d --- /dev/null +++ b/km-dist/docker/es_template_create.sh @@ -0,0 +1,661 @@ +echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}" +while true +do + curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 + if [ "$?" != "0" ];then + sleep 1s + else + echo "ElasticSearch Start Initialize" + break + fi +done + +curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_broker_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "NetworkProcessorAvgIdle" : { + "type" : "float" + }, + "UnderReplicatedPartitions" : { + "type" : "float" + }, + "BytesIn_min_15" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "RequestHandlerAvgIdle" : { + "type" : "float" + }, + "connectionsCount" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "TotalRequestQueueSize" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "TotalResponseQueueSize" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "Connections" : { + "type" : "double" + }, + "BytesIn_min_15" : { + "type" : "double" + }, + "PartitionURP" : { + "type" : "double" + }, + "HealthScore_Topics" : { + "type" : "double" + }, + "EventQueueSize" : { + "type" : "double" + }, + "ActiveControllerCount" : { + "type" : "double" + }, + "GroupDeads" : { + "type" : "double" + }, + "BytesIn_min_5" : { + "type" : "double" + }, + "HealthCheckTotal_Topics" : { + "type" : "double" + }, + "Partitions" : { + "type" : "double" + }, + "BytesOut" : { + "type" : "double" + }, + "Groups" : { + "type" : "double" + }, + "BytesOut_min_15" : { + "type" : "double" + }, + "TotalRequestQueueSize" : { + "type" : "double" + }, + "HealthCheckPassed_Groups" : { + "type" : "double" + }, + "TotalProduceRequests" : { + "type" : "double" + }, + "HealthCheckPassed" : { + "type" : "double" + }, + "TotalLogSize" : { + "type" : "double" + }, + "GroupEmptys" : { + "type" : "double" + }, + "PartitionNoLeader" : { + "type" : "double" + }, + "HealthScore_Brokers" : { + "type" : "double" + }, + "Messages" : { + "type" : "double" + }, + "Topics" : { + "type" : "double" + }, + "PartitionMinISR_E" : { + "type" : "double" + }, + "HealthCheckTotal" : { + "type" : "double" + }, + "Brokers" : { + "type" : "double" + }, + "Replicas" : { + "type" : "double" + }, + "HealthCheckTotal_Groups" : { + "type" : "double" + }, + "GroupRebalances" : { + "type" : "double" + }, + "MessageIn" : { + "type" : "double" + }, + "HealthScore" : { + "type" : "double" + }, + "HealthCheckPassed_Topics" : { + "type" : "double" + }, + "HealthCheckTotal_Brokers" : { + "type" : "double" + }, + "PartitionMinISR_S" : { + "type" : "double" + }, + "BytesIn" : { + "type" : "double" + }, + "BytesOut_min_5" : { + "type" : "double" + }, + "GroupActives" : { + "type" : "double" + }, + "MessagesIn" : { + "type" : "double" + }, + "GroupReBalances" : { + "type" : "double" + }, + "HealthCheckPassed_Brokers" : { + "type" : "double" + }, + "HealthScore_Groups" : { + "type" : "double" + }, + "TotalResponseQueueSize" : { + "type" : "double" + }, + "Zookeepers" : { + "type" : "double" + }, + "LeaderMessages" : { + "type" : "double" + }, + "HealthScore_Cluster" : { + "type" : "double" + }, + "HealthCheckPassed_Cluster" : { + "type" : "double" + }, + "HealthCheckTotal_Cluster" : { + "type" : "double" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "type" : "date" + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_group_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "group" : { + "type" : "keyword" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "HealthScore" : { + "type" : "float" + }, + "Lag" : { + "type" : "float" + }, + "OffsetConsumed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "groupMetric" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }[root@10-255-0-23 template]# cat ks_kafka_replication_metric +PUT _template/ks_kafka_replication_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_replication_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_topic_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "topic" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "BytesIn_min_15" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "BytesRejected" : { + "type" : "float" + }, + "PartitionURP" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ReplicationCount" : { + "type" : "float" + }, + "ReplicationBytesOut" : { + "type" : "float" + }, + "ReplicationBytesIn" : { + "type" : "float" + }, + "FailedFetchRequests" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "LogSize" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "FailedProduceRequests" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "brokerAgg" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +for i in {0..6}; +do + logdate=_$(date -d "${i} day ago" +%Y-%m-%d) + curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ + exit 2 +done + +echo "ElasticSearch Initialize Success" From d9e4abc3de55edd45e6bad1606e767888ee47a5f Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:05:59 +0800 Subject: [PATCH 07/23] Create ks-start.sh --- km-dist/docker/ks-start.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 km-dist/docker/ks-start.sh diff --git a/km-dist/docker/ks-start.sh b/km-dist/docker/ks-start.sh new file mode 100644 index 00000000..5e46fb7c --- /dev/null +++ b/km-dist/docker/ks-start.sh @@ -0,0 +1,7 @@ +sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml +sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml + +java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /km-rest.jar --spring.config.location=/conf/application.yml From 1a3cbf7a9d3e9055387ad8f978d3dc4becbe70b2 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:07:04 +0800 Subject: [PATCH 08/23] Create knowstreaming.conf --- km-dist/docker/ui/knowstreaming.conf | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 km-dist/docker/ui/knowstreaming.conf diff --git a/km-dist/docker/ui/knowstreaming.conf b/km-dist/docker/ui/knowstreaming.conf new file mode 100644 index 00000000..5b051595 --- /dev/null +++ b/km-dist/docker/ui/knowstreaming.conf @@ -0,0 +1,46 @@ +server { + listen 80; + server_name localhost; + gzip on; + gzip_buffers 16 8k; + gzip_comp_level 4; + gzip_http_version 1.0; + gzip_min_length 1280; + gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; + gzip_vary on; + root /pub; + location / { + root /pub; + if ($request_filename ~* .*\.(?:htm|html|json)$) { + add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; + } + try_files $uri /layout/index.html; + } + location ~* \.(json)$ { + add_header Cache-Control no-cache; + } + location @kmfallback { + } + #location ~ ^/(clusters|config|cluster|login) { + # rewrite ^.*$ /; + #} + location ~ ^/ks-km/api/v3 { + #rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; + proxy_pass http://knowstreaming-manager; + #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; + #proxy_cookie_path /ks-km/ /; + #proxy_set_header Host $host; + #proxy_set_header Referer $http_referer; + #proxy_set_header Cookie $http_cookie; + #proxy_set_header X-Real-Ip $remote_addr; + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~ ^/logi-security/api/v1 { + #rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; + proxy_pass http://knowstreaming-manager; + #proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; + } + location ~ ^/(401|403|404|500){ + rewrite ^.*$ /; + } + } From 7ed0e7dd23a7422bb92c120b6d9eda8a1e0f9dd2 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:14:02 +0800 Subject: [PATCH 09/23] Create dockerfile --- km-dist/docker/ui/dockerfile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 km-dist/docker/ui/dockerfile diff --git a/km-dist/docker/ui/dockerfile b/km-dist/docker/ui/dockerfile new file mode 100644 index 00000000..dfe605dc --- /dev/null +++ b/km-dist/docker/ui/dockerfile @@ -0,0 +1,5 @@ +FROM nginx:1.18 + +ADD pub.tar / +ADD knowstreaming.conf /etc/nginx/conf.d/ +RUN rm -rf /etc/nginx/conf.d/default.conf From 35f1dd80820daf37c53425488c805dc25d10ec9c Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:14:47 +0800 Subject: [PATCH 10/23] Create dockerfile --- km-dist/docker/manager/dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 km-dist/docker/manager/dockerfile diff --git a/km-dist/docker/manager/dockerfile b/km-dist/docker/manager/dockerfile new file mode 100644 index 00000000..4d850f6e --- /dev/null +++ b/km-dist/docker/manager/dockerfile @@ -0,0 +1,17 @@ +FROM java:8 + +#维护者 +MAINTAINER wangdongfang + +# 编译好的knowstreaming包 +ADD km-rest.jar / +# docker启动脚本 helm安装不使用 +ADD ks-start.sh / +# es初始化脚本 helm安装不使用 +ADD es_template_create.sh / + +RUN mkdir /conf +# { 配置文件 helm安装不使用 +ADD application.yml /conf/ +ADD logback-spring.xml /conf/ +# } From bfd026cae72af19974cc31534e7d13c1733a9853 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:16:28 +0800 Subject: [PATCH 11/23] Create dockerfile --- km-dist/docker/mysql/dockerfile | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 km-dist/docker/mysql/dockerfile diff --git a/km-dist/docker/mysql/dockerfile b/km-dist/docker/mysql/dockerfile new file mode 100644 index 00000000..00ee9b9d --- /dev/null +++ b/km-dist/docker/mysql/dockerfile @@ -0,0 +1,4 @@ +FROM mysql/mysql-server:5.7.38 +COPY initsql /docker-entrypoint-initdb.d/ +COPY init.sh /docker-entrypoint-initdb.d/ +ADD my.cnf /etc/ From ec83db267e01b1f2226486e4c143628261756229 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:17:02 +0800 Subject: [PATCH 12/23] Create init.sh --- km-dist/docker/mysql/init.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 km-dist/docker/mysql/init.sh diff --git a/km-dist/docker/mysql/init.sh b/km-dist/docker/mysql/init.sh new file mode 100644 index 00000000..99248d67 --- /dev/null +++ b/km-dist/docker/mysql/init.sh @@ -0,0 +1,6 @@ +if [ "$MYSQL_DATABASE" ]; then + "${mysql[@]}" < /docker-entrypoint-initdb.d/initsql +else + echo "CREATE DATABASE IF NOT EXISTS ks ;" | "${mysql[@]}" + "${mysql[@]}" ks < /docker-entrypoint-initdb.d/initsql +fi From e5a582cfadc9074575ee39d6f906bf253f824900 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:17:25 +0800 Subject: [PATCH 13/23] Create my.cnf --- km-dist/docker/mysql/my.cnf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 km-dist/docker/mysql/my.cnf diff --git a/km-dist/docker/mysql/my.cnf b/km-dist/docker/mysql/my.cnf new file mode 100644 index 00000000..79f0db15 --- /dev/null +++ b/km-dist/docker/mysql/my.cnf @@ -0,0 +1,15 @@ +[client] +default-character-set=utf8 +[mysql] +default-character-set=utf8 +[mysqld] +skip-host-cache +skip-name-resolve +datadir=/data/mysql +socket=/var/lib/mysql/mysql.sock +secure-file-priv=/var/lib/mysql-files +character-set-server=utf8 +user=mysql +symbolic-links=0 +pid-file=/var/run/mysqld/mysqld.pid +sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION From 77b8c758dc070e7868a636ff257e894c23e25936 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:18:17 +0800 Subject: [PATCH 14/23] Create initsql --- km-dist/docker/mysql/initsql | 784 +++++++++++++++++++++++++++++++++++ 1 file changed, 784 insertions(+) create mode 100644 km-dist/docker/mysql/initsql diff --git a/km-dist/docker/mysql/initsql b/km-dist/docker/mysql/initsql new file mode 100644 index 00000000..d5916260 --- /dev/null +++ b/km-dist/docker/mysql/initsql @@ -0,0 +1,784 @@ +DROP TABLE IF EXISTS `ks_km_broker`; +CREATE TABLE `ks_km_broker` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '物理集群ID', + `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', + `host` varchar(128) NOT NULL DEFAULT '' COMMENT 'broker主机名', + `port` int(16) NOT NULL DEFAULT '-1' COMMENT 'broker端口', + `jmx_port` int(16) NOT NULL DEFAULT '-1' COMMENT 'Jmx端口', + `start_timestamp` bigint(20) NOT NULL DEFAULT '-1' COMMENT '启动时间', + `rack` varchar(128) NOT NULL DEFAULT '' COMMENT 'Rack信息', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '状态: 1存活，0未存活', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `endpoint_map` varchar(1024) NOT NULL DEFAULT '' COMMENT '监听信息', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_phy_id_broker_id` (`cluster_phy_id`,`broker_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Broker信息表'; + + + +DROP TABLE IF EXISTS `ks_km_broker_config`; +CREATE TABLE `ks_km_broker_config` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', + `config_name` varchar(192) NOT NULL DEFAULT '' COMMENT '配置名称', + `config_value` text COMMENT '配置值', + `diff_type` int(16) NOT NULL DEFAULT '-1' COMMENT '差异类型', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_broker_name` (`cluster_phy_id`,`broker_id`,`config_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Broker配置信息表'; + + + +DROP TABLE IF EXISTS `ks_km_cluster_balance_job`; +CREATE TABLE `ks_km_cluster_balance_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `brokers` varchar(1024) NOT NULL DEFAULT '' COMMENT '均衡节点', + `topic_black_list` varchar(4096) NOT NULL DEFAULT '' COMMENT 'topic黑名单', + `type` int(16) NOT NULL DEFAULT '0' COMMENT '1:周期均衡，2：立即均衡', + `balance_interval_json` text COMMENT '均衡区间详情', + `metric_calculation_period` int(16) NOT NULL DEFAULT '0' COMMENT '指标计算周期，单位分钟', + `reassignment_json` text COMMENT '迁移脚本', + `parallel_num` int(16) NOT NULL DEFAULT '0' COMMENT '任务并行数', + `execution_strategy` int(16) NOT NULL DEFAULT '0' COMMENT '执行策略， 1：优先最大副本，2：优先最小副本', + `throttle_unit_b` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', + `total_reassign_size` double NOT NULL DEFAULT '0' COMMENT '总迁移大小', + `total_reassign_replica_num` int(16) NOT NULL DEFAULT '0' COMMENT '总迁移副本数', + `move_in_topic_list` varchar(4096) NOT NULL DEFAULT '' COMMENT '移入topic', + `broker_balance_detail` text COMMENT '节点均衡详情', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态 1：进行中，2：准备，3，成功，4：失败，5：取消', + `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', + `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务完成时间', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + `description` text COMMENT '备注', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群均衡任务'; + + + +DROP TABLE IF EXISTS `ks_km_cluster_balance_job_config`; +CREATE TABLE `ks_km_cluster_balance_job_config` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `brokers` varchar(256) NOT NULL DEFAULT '' COMMENT '均衡节点', + `topic_black_list` varchar(4096) NOT NULL DEFAULT '' COMMENT 'topic黑名单', + `task_cron` varchar(64) NOT NULL DEFAULT '' COMMENT '任务周期', + `balance_interval_json` text COMMENT '均衡区间详情', + `metric_calculation_period` int(16) NOT NULL DEFAULT '0' COMMENT '指标计算周期，单位分钟', + `reassignment_json` text COMMENT '迁移脚本', + `parallel_num` int(16) NOT NULL DEFAULT '0' COMMENT '任务并行数', + `execution_strategy` int(16) NOT NULL DEFAULT '0' COMMENT '执行策略， 1：优先最大副本，2：优先最小副本', + `throttle_unit_b` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', + `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态 0：未开启，1：开启', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群均衡任务'; + + + +DROP TABLE IF EXISTS `ks_km_cluster_balance_reassign`; +CREATE TABLE `ks_km_cluster_balance_reassign` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `job_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '父任务ID', + `cluster_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', + `original_broker_ids` text COMMENT '源BrokerId列表', + `reassign_broker_ids` text COMMENT '目标BrokerId列表', + `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务完成时间', + `extend_data` text COMMENT '扩展数据', + `status` int(16) NOT NULL DEFAULT '2' COMMENT '任务状态', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='集群平衡迁移详情'; + + + +DROP TABLE IF EXISTS `ks_km_group_member`; +CREATE TABLE `ks_km_group_member` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `topic_name` varchar(192) NOT NULL DEFAULT '' COMMENT 'Topic名称', + `group_name` varchar(192) NOT NULL DEFAULT '' COMMENT 'Group名称', + `kafka_user` varchar(192) NOT NULL DEFAULT '' COMMENT 'Kafka用户', + `state` varchar(64) NOT NULL DEFAULT '' COMMENT '状态', + `member_count` int(11) NOT NULL DEFAULT '0' COMMENT '成员数', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_topic_group` (`cluster_phy_id`,`topic_name`,`group_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='GroupMember信息表'; + + + +DROP TABLE IF EXISTS `ks_km_health_check_result`; +CREATE TABLE `ks_km_health_check_result` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `dimension` int(11) NOT NULL DEFAULT '0' COMMENT '检查维度(0:未知，1:Cluster，2:Broker，3:Topic，4:Group)', + `config_name` varchar(192) NOT NULL DEFAULT '' COMMENT '配置名', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '物理集群ID', + `res_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', + `passed` int(11) NOT NULL DEFAULT '0' COMMENT '检查通过(0:未通过，1:通过)', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_dimension_config_cluster_res` (`dimension`,`config_name`,`cluster_phy_id`,`res_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='健康检查结果'; + + + +DROP TABLE IF EXISTS `ks_km_job`; +CREATE TABLE `ks_km_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', + `job_name` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务名称', + `job_type` tinyint(10) NOT NULL COMMENT '任务类型', + `job_status` tinyint(10) NOT NULL COMMENT '任务状态', + `job_data` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务的详细信息', + `job_desc` varchar(1024) NOT NULL DEFAULT '' COMMENT '任务描述', + `cluster_id` int(11) NOT NULL COMMENT 'kafka集群id', + `target` varchar(8192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '任务执行对象', + `running_status` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '任务运行详细状态(json), Success：7 Fail：1 Doing：2', + `creator` varchar(45) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '创建者', + `plan_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '计划执行时间', + `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '实际执行时间', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `index_cluster_id` (`cluster_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Job信息'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_acl`; +CREATE TABLE `ks_km_kafka_acl` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `principal` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Kafka用户', + `operation` int(11) NOT NULL DEFAULT '0' COMMENT '操作', + `permission_type` int(11) NOT NULL DEFAULT '0' COMMENT '权限类型(0:未知，1:任意，2:拒绝，3:允许)', + `host` varchar(192) NOT NULL DEFAULT '127.0.0.1' COMMENT '机器', + `resource_type` int(11) NOT NULL DEFAULT '0' COMMENT '资源类型(0:未知，1:任意，2:Topic，3:Group，4:Cluster，5:事务，6:Token)', + `resource_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', + `pattern_type` int(11) NOT NULL COMMENT '匹配类型(0:未知，1:任意，2:Match，3:Literal，4:prefixed)', + `unique_field` varchar(1024) NOT NULL DEFAULT '' COMMENT '唯一字段，由cluster_phy_id等字段组成', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_uniq_field` (`unique_field`), + KEY `idx_cluster_phy_id_principal_res_name` (`cluster_phy_id`,`principal`,`resource_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='ACL信息'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_change_record`; +CREATE TABLE `ks_km_kafka_change_record` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `res_type` int(11) NOT NULL DEFAULT '-1' COMMENT '资源类型', + `res_name` varchar(192) NOT NULL DEFAULT '' COMMENT '资源名称', + `operate_type` int(11) NOT NULL DEFAULT '-1' COMMENT '操作类型', + `operate_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '操作时间', + `unique_field` varchar(1024) NOT NULL DEFAULT '' COMMENT '唯一键字段', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `unique_field` (`unique_field`), + KEY `idx_cluster_phy_id` (`cluster_phy_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Kafka变更记录表'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_controller`; +CREATE TABLE `ks_km_kafka_controller` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群id', + `broker_id` int(16) NOT NULL DEFAULT '-1' COMMENT 'brokerId', + `broker_host` varchar(256) NOT NULL DEFAULT '' COMMENT '主机名', + `broker_rack` varchar(256) NOT NULL DEFAULT '' COMMENT 'BrokerRack信息', + `timestamp` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'controller变更时间，-1表示未存活', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_broker_timestamp` (`cluster_phy_id`,`broker_id`,`timestamp`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='controller记录表'; + + + +DROP TABLE IF EXISTS `ks_km_kafka_user`; +CREATE TABLE `ks_km_kafka_user` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '名称', + `token` varchar(8192) NOT NULL DEFAULT '' COMMENT '密钥', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_phy_id_name` (`cluster_phy_id`,`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Kafka-User信息表'; + + + +DROP TABLE IF EXISTS `ks_km_partition`; +CREATE TABLE `ks_km_partition` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', + `leader_broker_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区的LeaderBroker，-1表示无Leader', + `in_sync_replicas` varchar(512) NOT NULL DEFAULT '-1' COMMENT 'ISR', + `assign_replicas` varchar(512) NOT NULL DEFAULT '-1' COMMENT 'AR', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_topic_partition` (`cluster_phy_id`,`topic_name`,`partition_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Partition信息表'; + + + +DROP TABLE IF EXISTS `ks_km_physical_cluster`; +CREATE TABLE `ks_km_physical_cluster` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '集群id', + `name` varchar(128) NOT NULL DEFAULT '' COMMENT '集群名称', + `zookeeper` varchar(2048) NOT NULL DEFAULT '' COMMENT 'zk地址', + `bootstrap_servers` varchar(2048) NOT NULL DEFAULT '' COMMENT 'server地址', + `kafka_version` varchar(32) NOT NULL DEFAULT '' COMMENT 'kafka版本', + `client_properties` text COMMENT 'Kafka客户端配置', + `jmx_properties` text COMMENT 'JMX配置', + `description` text COMMENT '备注', + `auth_type` int(11) NOT NULL DEFAULT '0' COMMENT '认证类型，-1未知，0:无认证，', + `run_state` tinyint(4) NOT NULL DEFAULT '1' COMMENT '运行状态, 0表示未监控, 1监控中，有ZK，2:监控中，无ZK', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '接入时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='物理集群信息表'; + + + +DROP TABLE IF EXISTS `ks_km_platform_cluster_config`; +CREATE TABLE `ks_km_platform_cluster_config` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', + `cluster_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `value_group` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项组', + `value_name` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项名字', + `value` text COMMENT '配置项的值', + `description` text COMMENT '备注', + `operator` varchar(16) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '操作者', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_id_group_name` (`cluster_id`,`value_group`,`value_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='KS平台集群配置'; + + + +DROP TABLE IF EXISTS `ks_km_reassign_job`; +CREATE TABLE `ks_km_reassign_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `reassignment_json` text COMMENT '迁移计划', + `throttle_unit_byte` bigint(20) NOT NULL DEFAULT '0' COMMENT '限流值', + `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务完成时间', + `creator` varchar(64) NOT NULL DEFAULT '' COMMENT '操作人', + `description` text COMMENT '备注', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='迁移Job信息'; + + + +DROP TABLE IF EXISTS `ks_km_reassign_sub_job`; +CREATE TABLE `ks_km_reassign_sub_job` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增id', + `job_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '父任务ID', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '集群id', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `partition_id` int(11) NOT NULL DEFAULT '-1' COMMENT '分区ID', + `original_broker_ids` text COMMENT '源BrokerId列表', + `reassign_broker_ids` text COMMENT '目标BrokerId列表', + `start_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务开始时间', + `finished_time` timestamp NOT NULL DEFAULT '1971-1-1 00:00:00' COMMENT '任务完成时间', + `extend_data` text COMMENT '扩展数据', + `status` int(16) NOT NULL DEFAULT '0' COMMENT '任务状态', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '任务修改时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='迁移SubJob信息'; + + + +DROP TABLE IF EXISTS `ks_km_topic`; +CREATE TABLE `ks_km_topic` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `cluster_phy_id` bigint(20) NOT NULL DEFAULT '-1' COMMENT '集群ID', + `topic_name` varchar(192) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Topic名称', + `replica_num` int(11) NOT NULL DEFAULT '-1' COMMENT '副本数', + `partition_num` int(11) NOT NULL DEFAULT '-1' COMMENT '分区数', + `broker_ids` varchar(2048) NOT NULL DEFAULT '' COMMENT 'BrokerId列表', + `partition_map` text COMMENT '分区分布信息', + `retention_ms` bigint(20) NOT NULL DEFAULT '-2' COMMENT '保存时间，-2：未知，-1：无限制，>=0对应时间，单位ms', + `type` tinyint(4) NOT NULL DEFAULT '0' COMMENT 'Topic类型，默认0，0:普通，1:Kafka内部', + `description` text COMMENT '备注信息', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间(尽量与Topic实际创建时间一致)', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间(尽量与Topic实际创建时间一致)', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_cluster_phy_id_topic_name` (`cluster_phy_id`,`topic_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Topic信息表'; + + +DROP TABLE IF EXISTS `ks_km_app_node`; +CREATE TABLE `ks_km_app_node` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `host_name` varchar(256) NOT NULL DEFAULT '' COMMENT 'host', + `ip` varchar(256) NOT NULL DEFAULT '' COMMENT 'ip', + `beat_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'node 的心跳时间', + `app_name` varchar(128) NOT NULL DEFAULT '' COMMENT 'km 集群的应用名', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + KEY `idx_app_host` (`app_name`,`host_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='km集群部署的node信息'; + +-- Logi-Job模块的sql，安装KS-KM需要执行该sql + + +DROP TABLE IF EXISTS `logi_job`; +CREATE TABLE `logi_job` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', + `task_code` varchar(255) NOT NULL DEFAULT '' COMMENT '任务code', + `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', + `try_times` int(10) NOT NULL DEFAULT '0' COMMENT '第几次重试', + `worker_code` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `start_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '开始时间', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `job_code` (`job_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='正在执行的job信息'; + + + +DROP TABLE IF EXISTS `logi_job_log`; +CREATE TABLE `logi_job_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'job taskCode', + `task_code` varchar(255) NOT NULL DEFAULT '' COMMENT '任务code', + `task_name` varchar(255) NOT NULL DEFAULT '' COMMENT '任务名称', + `task_desc` varchar(255) NOT NULL DEFAULT '' COMMENT '任务描述', + `task_id` bigint(20) NOT NULL DEFAULT '0' COMMENT '任务id', + `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', + `try_times` int(10) NOT NULL DEFAULT '0' COMMENT '第几次重试', + `worker_code` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器', + `worker_ip` varchar(200) NOT NULL DEFAULT '' COMMENT '执行机器ip', + `start_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '开始时间', + `end_time` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '结束时间', + `status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '执行结果 1成功 2失败 3取消', + `error` text NOT NULL COMMENT '错误信息', + `result` text NOT NULL COMMENT '执行结果', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `index_job_code` (`job_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='job执行历史日志'; + + + +DROP TABLE IF EXISTS `logi_task`; +CREATE TABLE `logi_task` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', + `task_name` varchar(255) NOT NULL DEFAULT '' COMMENT '名称', + `task_desc` varchar(1000) NOT NULL DEFAULT '' COMMENT '任务描述', + `cron` varchar(100) NOT NULL DEFAULT '' COMMENT 'cron 表达式', + `class_name` varchar(255) NOT NULL DEFAULT '' COMMENT '类的全限定名', + `params` varchar(1000) NOT NULL DEFAULT '' COMMENT '执行参数 map 形式{key1:value1,key2:value2}', + `retry_times` int(10) NOT NULL DEFAULT '0' COMMENT '允许重试次数', + `last_fire_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '上次执行时间', + `timeout` bigint(20) NOT NULL DEFAULT '0' COMMENT '超时 毫秒', + `status` tinyint(4) NOT NULL DEFAULT '0' COMMENT '1等待 2运行中 3暂停', + `sub_task_codes` varchar(1000) NOT NULL DEFAULT '' COMMENT '子任务code列表,逗号分隔', + `consensual` varchar(200) NOT NULL DEFAULT '' COMMENT '执行策略', + `owner` varchar(200) NOT NULL DEFAULT '' COMMENT '责任人', + `task_worker_str` varchar(3000) NOT NULL DEFAULT '' COMMENT '机器执行信息', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `task_code` (`task_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='任务信息'; + + + +DROP TABLE IF EXISTS `logi_task_lock`; +CREATE TABLE `logi_task_lock` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'task taskCode', + `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `expire_time` bigint(20) NOT NULL DEFAULT '0' COMMENT '过期时间', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='任务锁'; + + + +DROP TABLE IF EXISTS `logi_worker`; +CREATE TABLE `logi_worker` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', + `worker_name` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker名', + `ip` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker的ip', + `cpu` int(11) NOT NULL DEFAULT '0' COMMENT 'cpu数量', + `cpu_used` double NOT NULL DEFAULT '0' COMMENT 'cpu使用率', + `memory` double NOT NULL DEFAULT '0' COMMENT '内存,以M为单位', + `memory_used` double NOT NULL DEFAULT '0' COMMENT '内存使用率', + `jvm_memory` double NOT NULL DEFAULT '0' COMMENT 'jvm堆大小，以M为单位', + `jvm_memory_used` double NOT NULL DEFAULT '0' COMMENT 'jvm堆使用率', + `job_num` int(10) NOT NULL DEFAULT '0' COMMENT '正在执行job数', + `heartbeat` datetime DEFAULT '1971-01-01 00:00:00' COMMENT '心跳时间', + `app_name` varchar(100) NOT NULL DEFAULT '' COMMENT '被调度的应用名称', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `worker_code` (`worker_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker信息'; + + + +DROP TABLE IF EXISTS `logi_worker_blacklist`; +CREATE TABLE `logi_worker_blacklist` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `worker_code` varchar(100) NOT NULL DEFAULT '' COMMENT 'worker taskCode', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `worker_code` (`worker_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='worker黑名单列表'; + +DROP TABLE IF EXISTS `logi_security_dept`; +CREATE TABLE `logi_security_dept` +( + id int auto_increment primary key, + dept_name varchar(10) not null comment '部门名', + parent_id int not null comment '父部门id', + leaf tinyint(1) not null comment '是否叶子部门', + level tinyint not null comment 'parentId为0的层级为1', + description varchar(20) null comment '描述', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '部门信息表'; + +DROP TABLE IF EXISTS `logi_security_message`; +CREATE TABLE `logi_security_message` +( + id int auto_increment primary key, + title varchar(60) not null comment '标题', + content varchar(256) null comment '内容', + read_tag tinyint(1) default 0 null comment '是否已读', + oplog_id int null comment '操作日志id', + user_id int null comment '这条消息属于哪个用户的，用户id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '消息中心'; + +DROP TABLE IF EXISTS `logi_security_oplog`; +CREATE TABLE `logi_security_oplog` +( + id int auto_increment primary key, + operator_ip varchar(64) not null comment '操作者ip', + operator varchar(64) null comment '操作者账号', + operate_page varchar(64) not null default '' comment '操作页面', + operate_type varchar(64) not null comment '操作类型', + target_type varchar(64) not null comment '对象分类', + target varchar(1024) not null comment '操作对象', + operation_methods varchar(64) not null default '' comment '操作方式', + detail text null comment '日志详情', + create_time timestamp default CURRENT_TIMESTAMP null, + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '操作日志'; + + +DROP TABLE IF EXISTS `logi_security_oplog_extra`; +CREATE TABLE `logi_security_oplog_extra` +( + id int auto_increment primary key, + info varchar(16) null comment '信息', + type tinyint not null comment '哪种信息：1：操作页面;2：操作类型;3：对象分类', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '操作日志信息（操作页面、操作类型、对象分类）'; + +DROP TABLE IF EXISTS `logi_security_permission`; +CREATE TABLE `logi_security_permission` +( + id int auto_increment primary key, + permission_name varchar(40) not null comment '权限名字', + parent_id int not null comment '父权限id', + leaf tinyint(1) not null comment '是否叶子权限点（具体的操作）', + level tinyint not null comment '权限点的层级（parentId为0的层级为1）', + description varchar(64) null comment '权限点描述', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '权限表'; + +DROP TABLE IF EXISTS `logi_security_project`; +CREATE TABLE `logi_security_project` +( + id int auto_increment comment '项目id' primary key, + project_code varchar(128) not null comment '项目编号', + project_name varchar(128) not null comment '项目名', + description varchar(512) default '' not null comment '项目描述', + dept_id int not null comment '部门id', + running tinyint(1) default 1 not null comment '启用 or 停用', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '项目表'; + +DROP TABLE IF EXISTS `logi_security_resource_type`; +CREATE TABLE `logi_security_resource_type` +( + id int auto_increment primary key, + type_name varchar(16) null comment '资源类型名', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '资源类型表'; + +DROP TABLE IF EXISTS `logi_security_role`; +CREATE TABLE `logi_security_role` +( + id int auto_increment primary key, + role_code varchar(128) not null comment '角色编号', + role_name varchar(128) not null comment '名称', + description varchar(128) null comment '角色描述', + last_reviser varchar(30) null comment '最后修改人', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '角色信息'; + +DROP TABLE IF EXISTS `logi_security_role_permission`; +CREATE TABLE `logi_security_role_permission` +( + id int auto_increment primary key, + role_id int not null comment '角色id', + permission_id int not null comment '权限id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '角色权限表（只保留叶子权限与角色关系）'; + +DROP TABLE IF EXISTS `logi_security_user`; +CREATE TABLE `logi_security_user` +( + id int auto_increment primary key, + user_name varchar(64) not null comment '用户账号', + pw varchar(2048) not null comment '用户密码', + salt char(5) default '' not null comment '密码盐', + real_name varchar(128) default '' not null comment '真实姓名', + phone char(11) default '' not null comment 'mobile', + email varchar(30) default '' not null comment 'email', + dept_id int null comment '所属部门id', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + create_time timestamp default CURRENT_TIMESTAMP null comment '注册时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户信息'; + +DROP TABLE IF EXISTS `logi_security_user_project`; +CREATE TABLE `logi_security_user_project` +( + id int auto_increment primary key, + user_id int not null comment '用户id', + project_id int not null comment '项目id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户项目关系表（项目负责人）'; + +DROP TABLE IF EXISTS `logi_security_user_resource`; +CREATE TABLE `logi_security_user_resource` +( + id int auto_increment primary key, + user_id int not null comment '用户id', + project_id int not null comment '资源所属项目id', + resource_type_id int not null comment '资源类别id', + resource_id int not null comment '资源id', + control_level tinyint not null comment '管理级别：1（查看权限）2（管理权限）', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户和资源关系表'; + +DROP TABLE IF EXISTS `logi_security_user_role`; +CREATE TABLE `logi_security_user_role` +( + id int auto_increment primary key, + user_id int not null comment '用户id', + role_id int not null comment '角色id', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null on update CURRENT_TIMESTAMP comment '更新时间', + is_delete tinyint(1) default 0 not null comment '逻辑删除', + app_name varchar(16) null comment '应用名称' +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 comment '用户角色表'; + + +DROP TABLE IF EXISTS `logi_security_config`; +CREATE TABLE `logi_security_config` +( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键自增', + `value_group` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项组', + `value_name` varchar(100) NOT NULL DEFAULT '' COMMENT '配置项名字', + `value` text COMMENT '配置项的值', + `edit` int(4) NOT NULL DEFAULT '1' COMMENT '是否可以编辑 1 不可编辑（程序获取） 2 可编辑', + `status` int(4) NOT NULL DEFAULT '1' COMMENT '1 正常 2 禁用', + `memo` varchar(1000) NOT NULL DEFAULT '' COMMENT '备注', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + `is_delete` tinyint(1) NOT NULL DEFAULT '0' COMMENT '逻辑删除', + `app_name` varchar(16) COLLATE utf8_bin DEFAULT NULL COMMENT '应用名称', + `operator` varchar(16) COLLATE utf8_bin DEFAULT NULL COMMENT '操作者', + PRIMARY KEY (`id`), + KEY `idx_group_name` (`value_group`,`value_name`) +) ENGINE=InnoDB AUTO_INCREMENT=1592 DEFAULT CHARSET=utf8 COMMENT='logi配置项'; + + +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_CLUSTER_NO_CONTROLLER','{ \"value\": 1, \"weight\": 30 } ','集群Controller数正常','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_BROKER_REQUEST_QUEUE_FULL','{ \"value\": 10, \"weight\": 20 } ','Broker-RequestQueueSize指标','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_BROKER_NETWORK_PROCESSOR_AVG_IDLE_TOO_LOW','{ \"value\": 0.8, \"weight\": 20 } ','Broker-NetworkProcessorAvgIdlePercent指标','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_GROUP_RE_BALANCE_TOO_FREQUENTLY','{\n \"latestMinutes\": 10,\n \"detectedTimes\": 8,\n \"weight\": 10\n}\n','Group的re-balance频率','know-streaming'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_TOPIC_NO_LEADER','{ \"value\": 1, \"weight\": 10 } ','Topic 无Leader数','know-stream'); +INSERT INTO `ks_km_platform_cluster_config` (`cluster_id`,`value_group`,`value_name`,`value`,`description`,`operator`) VALUES (-1,'HEALTH','HC_TOPIC_UNDER_REPLICA_TOO_LONG','{ \"latestMinutes\": 10, \"detectedTimes\": 8, \"weight\": 10 } ','Topic 未同步持续时间','know-streaming'); +-- 初始化权限 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1593', '多集群管理', '0', '0', '1', '多集群管理', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1595', '系统管理', '0', '0', '1', '系统管理', '0', 'know-streaming'); + +-- 多集群管理权限 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1597', '接入集群', '1593', '1', '2', '接入集群', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1599', '删除集群', '1593', '1', '2', '删除集群', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1601', 'Cluster-修改集群信息', '1593', '1', '2', 'Cluster-修改集群信息', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1603', 'Cluster-修改健康规则', '1593', '1', '2', 'Cluster-修改健康规则', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1605', 'Broker-修改Broker配置', '1593', '1', '2', 'Broker-修改Broker配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1607', 'Topic-新增Topic', '1593', '1', '2', 'Topic-新增Topic', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1609', 'Topic-扩分区', '1593', '1', '2', 'Topic-扩分区', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1611', 'Topic-删除Topic', '1593', '1', '2', 'Topic-扩分区', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1613', 'Topic-重置Offset', '1593', '1', '2', 'Topic-重置Offset', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1615', 'Topic-修改Topic配置', '1593', '1', '2', 'Topic-修改Topic配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1617', 'Consumers-重置Offset', '1593', '1', '2', 'Consumers-重置Offset', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1619', 'Test-Producer', '1593', '1', '2', 'Test-Producer', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1621', 'Test-Consumer', '1593', '1', '2', 'Test-Consumer', '0', 'know-streaming'); + +-- 系统管理权限 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1623', '配置管理-新增配置', '1595', '1', '2', '配置管理-新增配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1625', '配置管理-编辑配置', '1595', '1', '2', '配置管理-编辑配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1627', '配置管理-删除配置', '1595', '1', '2', '配置管理-删除配置', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1629', '用户管理-新增人员', '1595', '1', '2', '用户管理-新增人员', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1631', '用户管理-编辑人员', '1595', '1', '2', '用户管理-编辑人员', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1633', '用户管理-修改人员密码', '1595', '1', '2', '用户管理-修改人员密码', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1635', '用户管理-删除人员', '1595', '1', '2', '用户管理-删除人员', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1637', '用户管理-新增角色', '1595', '1', '2', '用户管理-新增角色', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1639', '用户管理-编辑角色', '1595', '1', '2', '用户管理-编辑角色', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1641', '用户管理-分配用户角色', '1595', '1', '2', '用户管理-分配用户角色', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('1643', '用户管理-删除角色', '1595', '1', '2', '用户管理-删除角色', '0', 'know-streaming'); + +-- 多集群管理权限2022-09-06新增 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2000', '多集群管理查看', '1593', '1', '2', '多集群管理查看', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2002', 'Topic-迁移副本', '1593', '1', '2', 'Topic-迁移副本', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2004', 'Topic-扩缩副本', '1593', '1', '2', 'Topic-扩缩副本', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2006', 'Cluster-LoadReBalance-周期均衡', '1593', '1', '2', 'Cluster-LoadReBalance-周期均衡', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2008', 'Cluster-LoadReBalance-立即均衡', '1593', '1', '2', 'Cluster-LoadReBalance-立即均衡', '0', 'know-streaming'); +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('2010', 'Cluster-LoadReBalance-设置集群规格', '1593', '1', '2', 'Cluster-LoadReBalance-设置集群规格', '0', 'know-streaming'); + + +-- 系统管理权限2022-09-06新增 +INSERT INTO `logi_security_permission` (`id`, `permission_name`, `parent_id`, `leaf`, `level`, `description`, `is_delete`, `app_name`) VALUES ('3000', '系统管理查看', '1595', '1', '2', '系统管理查看', '0', 'know-streaming'); + + + + + + +-- 初始化用户 +INSERT INTO `logi_security_user` (`id`, `user_name`, `pw`, `real_name`, `is_delete`, `app_name`) VALUES ('1', 'admin', 'V1ZkU2RHRlhOVGRSUmxweFUycFNhR0V6ZEdKSk1FRjRVVU5PWkdaVmJ6SlZiWGh6WVVWQ09YdEFWbXBLTkdGcmUxc2pRREpBSTExOVNqWlNiR3hvUUgwPXtAVmpKNGFre1sjQDNAI119SjZSbGxoQH0=Mv{#cdRgJ45Lqx}3IubEW87!==', '系统管理员', '0', 'know-streaming'); + +-- 初始化角色 +INSERT INTO `logi_security_role` (`id`, `role_code`, `role_name`, `description`, `last_reviser`, `is_delete`, `app_name`) VALUES ('1677', 'r15477137', '管理员角色', '包含系统所有权限', 'admin', '0', 'know-streaming'); + +-- 初始化角色权限关系 +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1597', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1599', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1601', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1603', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1605', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1607', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1609', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1611', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1613', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1615', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1617', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1619', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1621', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1593', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1623', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1625', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1627', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1629', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1631', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1633', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1635', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1637', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1639', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1641', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1643', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '1595', '0', 'know-streaming'); + +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2000', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2002', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2004', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2006', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2008', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '2010', '0', 'know-streaming'); +INSERT INTO `logi_security_role_permission` (`role_id`, `permission_id`, `is_delete`, `app_name`) VALUES ('1677', '3000', '0', 'know-streaming'); + +-- 初始化 用户角色关系 +INSERT INTO `logi_security_user_role` (`id`, `user_id`, `role_id`, `is_delete`, `app_name`) VALUES ('1', '1', '1677', '0', 'know-streaming'); + +INSERT INTO `logi_security_config` +(`value_group`,`value_name`,`value`,`edit`,`status`,`memo`,`is_delete`,`app_name`,`operator`) +VALUES +('SECURITY.LOGIN','SECURITY.TRICK_USERS','[\n \"admin\"\n]',1,1,'允许跳过登录的用户',0,'know-streaming','admin'); From 6cb1825d964410da5e482e05270b7a63c630fc26 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:19:10 +0800 Subject: [PATCH 15/23] Create es_template_create.sh --- km-dist/docker/manager/es_template_create.sh | 661 +++++++++++++++++++ 1 file changed, 661 insertions(+) create mode 100644 km-dist/docker/manager/es_template_create.sh diff --git a/km-dist/docker/manager/es_template_create.sh b/km-dist/docker/manager/es_template_create.sh new file mode 100644 index 00000000..615bf54d --- /dev/null +++ b/km-dist/docker/manager/es_template_create.sh @@ -0,0 +1,661 @@ +echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}" +while true +do + curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 + if [ "$?" != "0" ];then + sleep 1s + else + echo "ElasticSearch Start Initialize" + break + fi +done + +curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_broker_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "NetworkProcessorAvgIdle" : { + "type" : "float" + }, + "UnderReplicatedPartitions" : { + "type" : "float" + }, + "BytesIn_min_15" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "RequestHandlerAvgIdle" : { + "type" : "float" + }, + "connectionsCount" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "TotalRequestQueueSize" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + }, + "TotalResponseQueueSize" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_cluster_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "Connections" : { + "type" : "double" + }, + "BytesIn_min_15" : { + "type" : "double" + }, + "PartitionURP" : { + "type" : "double" + }, + "HealthScore_Topics" : { + "type" : "double" + }, + "EventQueueSize" : { + "type" : "double" + }, + "ActiveControllerCount" : { + "type" : "double" + }, + "GroupDeads" : { + "type" : "double" + }, + "BytesIn_min_5" : { + "type" : "double" + }, + "HealthCheckTotal_Topics" : { + "type" : "double" + }, + "Partitions" : { + "type" : "double" + }, + "BytesOut" : { + "type" : "double" + }, + "Groups" : { + "type" : "double" + }, + "BytesOut_min_15" : { + "type" : "double" + }, + "TotalRequestQueueSize" : { + "type" : "double" + }, + "HealthCheckPassed_Groups" : { + "type" : "double" + }, + "TotalProduceRequests" : { + "type" : "double" + }, + "HealthCheckPassed" : { + "type" : "double" + }, + "TotalLogSize" : { + "type" : "double" + }, + "GroupEmptys" : { + "type" : "double" + }, + "PartitionNoLeader" : { + "type" : "double" + }, + "HealthScore_Brokers" : { + "type" : "double" + }, + "Messages" : { + "type" : "double" + }, + "Topics" : { + "type" : "double" + }, + "PartitionMinISR_E" : { + "type" : "double" + }, + "HealthCheckTotal" : { + "type" : "double" + }, + "Brokers" : { + "type" : "double" + }, + "Replicas" : { + "type" : "double" + }, + "HealthCheckTotal_Groups" : { + "type" : "double" + }, + "GroupRebalances" : { + "type" : "double" + }, + "MessageIn" : { + "type" : "double" + }, + "HealthScore" : { + "type" : "double" + }, + "HealthCheckPassed_Topics" : { + "type" : "double" + }, + "HealthCheckTotal_Brokers" : { + "type" : "double" + }, + "PartitionMinISR_S" : { + "type" : "double" + }, + "BytesIn" : { + "type" : "double" + }, + "BytesOut_min_5" : { + "type" : "double" + }, + "GroupActives" : { + "type" : "double" + }, + "MessagesIn" : { + "type" : "double" + }, + "GroupReBalances" : { + "type" : "double" + }, + "HealthCheckPassed_Brokers" : { + "type" : "double" + }, + "HealthScore_Groups" : { + "type" : "double" + }, + "TotalResponseQueueSize" : { + "type" : "double" + }, + "Zookeepers" : { + "type" : "double" + }, + "LeaderMessages" : { + "type" : "double" + }, + "HealthScore_Cluster" : { + "type" : "double" + }, + "HealthCheckPassed_Cluster" : { + "type" : "double" + }, + "HealthCheckTotal_Cluster" : { + "type" : "double" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "type" : "date" + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_group_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "group" : { + "type" : "keyword" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "HealthScore" : { + "type" : "float" + }, + "Lag" : { + "type" : "float" + }, + "OffsetConsumed" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "groupMetric" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_partition_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "partitionId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "clusterPhyId" : { + "type" : "long" + }, + "topic" : { + "type" : "keyword" + }, + "metrics" : { + "properties" : { + "LogStartOffset" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "LogEndOffset" : { + "type" : "float" + } + } + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }[root@10-255-0-23 template]# cat ks_kafka_replication_metric +PUT _template/ks_kafka_replication_metric +{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_replication_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ + "order" : 10, + "index_patterns" : [ + "ks_kafka_topic_metric*" + ], + "settings" : { + "index" : { + "number_of_shards" : "10" + } + }, + "mappings" : { + "properties" : { + "brokerId" : { + "type" : "long" + }, + "routingValue" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "topic" : { + "type" : "keyword" + }, + "clusterPhyId" : { + "type" : "long" + }, + "metrics" : { + "properties" : { + "BytesIn_min_15" : { + "type" : "float" + }, + "Messages" : { + "type" : "float" + }, + "BytesRejected" : { + "type" : "float" + }, + "PartitionURP" : { + "type" : "float" + }, + "HealthCheckTotal" : { + "type" : "float" + }, + "ReplicationCount" : { + "type" : "float" + }, + "ReplicationBytesOut" : { + "type" : "float" + }, + "ReplicationBytesIn" : { + "type" : "float" + }, + "FailedFetchRequests" : { + "type" : "float" + }, + "BytesIn_min_5" : { + "type" : "float" + }, + "HealthScore" : { + "type" : "float" + }, + "LogSize" : { + "type" : "float" + }, + "BytesOut" : { + "type" : "float" + }, + "BytesOut_min_15" : { + "type" : "float" + }, + "FailedProduceRequests" : { + "type" : "float" + }, + "BytesIn" : { + "type" : "float" + }, + "BytesOut_min_5" : { + "type" : "float" + }, + "MessagesIn" : { + "type" : "float" + }, + "TotalProduceRequests" : { + "type" : "float" + }, + "HealthCheckPassed" : { + "type" : "float" + } + } + }, + "brokerAgg" : { + "type" : "keyword" + }, + "key" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "timestamp" : { + "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", + "index" : true, + "type" : "date", + "doc_values" : true + } + } + }, + "aliases" : { } + }' + +for i in {0..6}; +do + logdate=_$(date -d "${i} day ago" +%Y-%m-%d) + curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ + exit 2 +done + +echo "ElasticSearch Initialize Success" From a23819c3083c794340a37a48eb63e809fd0aef92 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:19:35 +0800 Subject: [PATCH 16/23] Create ks-start.sh --- km-dist/docker/manager/ks-start.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 km-dist/docker/manager/ks-start.sh diff --git a/km-dist/docker/manager/ks-start.sh b/km-dist/docker/manager/ks-start.sh new file mode 100644 index 00000000..5e46fb7c --- /dev/null +++ b/km-dist/docker/manager/ks-start.sh @@ -0,0 +1,7 @@ +sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml +sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml + +java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /km-rest.jar --spring.config.location=/conf/application.yml From 9b7b9a7af0e708d87c8a84dcd5a788a78b04df74 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:26:16 +0800 Subject: [PATCH 17/23] Delete es_template_create.sh --- km-dist/docker/es_template_create.sh | 661 --------------------------- 1 file changed, 661 deletions(-) delete mode 100644 km-dist/docker/es_template_create.sh diff --git a/km-dist/docker/es_template_create.sh b/km-dist/docker/es_template_create.sh deleted file mode 100644 index 615bf54d..00000000 --- a/km-dist/docker/es_template_create.sh +++ /dev/null @@ -1,661 +0,0 @@ -echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}" -while true -do - curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1 - if [ "$?" != "0" ];then - sleep 1s - else - echo "ElasticSearch Start Initialize" - break - fi -done - -curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_broker_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "NetworkProcessorAvgIdle" : { - "type" : "float" - }, - "UnderReplicatedPartitions" : { - "type" : "float" - }, - "BytesIn_min_15" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "RequestHandlerAvgIdle" : { - "type" : "float" - }, - "connectionsCount" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "TotalRequestQueueSize" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - }, - "TotalResponseQueueSize" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_cluster_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "Connections" : { - "type" : "double" - }, - "BytesIn_min_15" : { - "type" : "double" - }, - "PartitionURP" : { - "type" : "double" - }, - "HealthScore_Topics" : { - "type" : "double" - }, - "EventQueueSize" : { - "type" : "double" - }, - "ActiveControllerCount" : { - "type" : "double" - }, - "GroupDeads" : { - "type" : "double" - }, - "BytesIn_min_5" : { - "type" : "double" - }, - "HealthCheckTotal_Topics" : { - "type" : "double" - }, - "Partitions" : { - "type" : "double" - }, - "BytesOut" : { - "type" : "double" - }, - "Groups" : { - "type" : "double" - }, - "BytesOut_min_15" : { - "type" : "double" - }, - "TotalRequestQueueSize" : { - "type" : "double" - }, - "HealthCheckPassed_Groups" : { - "type" : "double" - }, - "TotalProduceRequests" : { - "type" : "double" - }, - "HealthCheckPassed" : { - "type" : "double" - }, - "TotalLogSize" : { - "type" : "double" - }, - "GroupEmptys" : { - "type" : "double" - }, - "PartitionNoLeader" : { - "type" : "double" - }, - "HealthScore_Brokers" : { - "type" : "double" - }, - "Messages" : { - "type" : "double" - }, - "Topics" : { - "type" : "double" - }, - "PartitionMinISR_E" : { - "type" : "double" - }, - "HealthCheckTotal" : { - "type" : "double" - }, - "Brokers" : { - "type" : "double" - }, - "Replicas" : { - "type" : "double" - }, - "HealthCheckTotal_Groups" : { - "type" : "double" - }, - "GroupRebalances" : { - "type" : "double" - }, - "MessageIn" : { - "type" : "double" - }, - "HealthScore" : { - "type" : "double" - }, - "HealthCheckPassed_Topics" : { - "type" : "double" - }, - "HealthCheckTotal_Brokers" : { - "type" : "double" - }, - "PartitionMinISR_S" : { - "type" : "double" - }, - "BytesIn" : { - "type" : "double" - }, - "BytesOut_min_5" : { - "type" : "double" - }, - "GroupActives" : { - "type" : "double" - }, - "MessagesIn" : { - "type" : "double" - }, - "GroupReBalances" : { - "type" : "double" - }, - "HealthCheckPassed_Brokers" : { - "type" : "double" - }, - "HealthScore_Groups" : { - "type" : "double" - }, - "TotalResponseQueueSize" : { - "type" : "double" - }, - "Zookeepers" : { - "type" : "double" - }, - "LeaderMessages" : { - "type" : "double" - }, - "HealthScore_Cluster" : { - "type" : "double" - }, - "HealthCheckPassed_Cluster" : { - "type" : "double" - }, - "HealthCheckTotal_Cluster" : { - "type" : "double" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "type" : "date" - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_group_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "group" : { - "type" : "keyword" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "HealthScore" : { - "type" : "float" - }, - "Lag" : { - "type" : "float" - }, - "OffsetConsumed" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "groupMetric" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_partition_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "partitionId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "clusterPhyId" : { - "type" : "long" - }, - "topic" : { - "type" : "keyword" - }, - "metrics" : { - "properties" : { - "LogStartOffset" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "LogEndOffset" : { - "type" : "float" - } - } - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }[root@10-255-0-23 template]# cat ks_kafka_replication_metric -PUT _template/ks_kafka_replication_metric -{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_replication_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ - "order" : 10, - "index_patterns" : [ - "ks_kafka_topic_metric*" - ], - "settings" : { - "index" : { - "number_of_shards" : "10" - } - }, - "mappings" : { - "properties" : { - "brokerId" : { - "type" : "long" - }, - "routingValue" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "topic" : { - "type" : "keyword" - }, - "clusterPhyId" : { - "type" : "long" - }, - "metrics" : { - "properties" : { - "BytesIn_min_15" : { - "type" : "float" - }, - "Messages" : { - "type" : "float" - }, - "BytesRejected" : { - "type" : "float" - }, - "PartitionURP" : { - "type" : "float" - }, - "HealthCheckTotal" : { - "type" : "float" - }, - "ReplicationCount" : { - "type" : "float" - }, - "ReplicationBytesOut" : { - "type" : "float" - }, - "ReplicationBytesIn" : { - "type" : "float" - }, - "FailedFetchRequests" : { - "type" : "float" - }, - "BytesIn_min_5" : { - "type" : "float" - }, - "HealthScore" : { - "type" : "float" - }, - "LogSize" : { - "type" : "float" - }, - "BytesOut" : { - "type" : "float" - }, - "BytesOut_min_15" : { - "type" : "float" - }, - "FailedProduceRequests" : { - "type" : "float" - }, - "BytesIn" : { - "type" : "float" - }, - "BytesOut_min_5" : { - "type" : "float" - }, - "MessagesIn" : { - "type" : "float" - }, - "TotalProduceRequests" : { - "type" : "float" - }, - "HealthCheckPassed" : { - "type" : "float" - } - } - }, - "brokerAgg" : { - "type" : "keyword" - }, - "key" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "timestamp" : { - "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis", - "index" : true, - "type" : "date", - "doc_values" : true - } - } - }, - "aliases" : { } - }' - -for i in {0..6}; -do - logdate=_$(date -d "${i} day ago" +%Y-%m-%d) - curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ - exit 2 -done - -echo "ElasticSearch Initialize Success" From e71e29391b046eee9ead21a662c26b7eaee0b8fb Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:26:24 +0800 Subject: [PATCH 18/23] Delete ks-start.sh --- km-dist/docker/ks-start.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 km-dist/docker/ks-start.sh diff --git a/km-dist/docker/ks-start.sh b/km-dist/docker/ks-start.sh deleted file mode 100644 index 5e46fb7c..00000000 --- a/km-dist/docker/ks-start.sh +++ /dev/null @@ -1,7 +0,0 @@ -sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml -sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml -sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml - -java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /km-rest.jar --spring.config.location=/conf/application.yml From 3e12ba34f7a545f47bc22bd659e7fb08bd5ef168 Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:33:05 +0800 Subject: [PATCH 19/23] Update docker-compose.yml --- km-dist/docker/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index f546db77..219ceb5c 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: "2" services: # *不要调整knowstreaming-manager服务名称，ui中会用到 knowstreaming-manager: - image: knowstreaming/knowstreaming-manager:0.2.0-test + image: knowstreaming/knowstreaming-manager:latest container_name: knowstreaming-manager privileged: true restart: always @@ -35,7 +35,7 @@ services: # volumes: # - /ks/manage/log:/logs knowstreaming-ui: - image: knowstreaming/knowstreaming-ui:0.2.0-test1 + image: knowstreaming/knowstreaming-ui:latest container_name: knowstreaming-ui restart: always ports: @@ -69,7 +69,7 @@ services: # es初始化服务，与manager使用同一镜像 # 首次启动es需初始化模版和索引,后续会自动创建 knowstreaming-init: - image: knowstreaming/knowstreaming-manager:0.2.0-test + image: knowstreaming/knowstreaming-manager:latest container_name: knowstreaming-init depends_on: - elasticsearch-single @@ -82,7 +82,7 @@ services: SERVER_ES_ADDRESS: elasticsearch-single:9200 knowstreaming-mysql: - image: knowstreaming/knowstreaming-mysql:0.2.0-test + image: knowstreaming/knowstreaming-mysql:latest container_name: knowstreaming-mysql restart: always environment: From 0c18c5b4f6203487c06f9be87a0f4360426eb8af Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:43:23 +0800 Subject: [PATCH 20/23] =?UTF-8?q?Update=20=E5=8D=95=E6=9C=BA=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=89=8B=E5=86=8C.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index bc895e10..713b297f 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -98,7 +98,7 @@ helm pull knowstreaming/knowstreaming-manager **安装命令** ```bash -# `v3.0.0-beta.2`版本开始，默认账号密码为`admin` / `admin`； +# `v3.0.0-beta.3`版本开始(docker镜像为0.3.0版本开始)，默认账号密码为`admin` / `admin`； # https://hub.docker.com/u/knowstreaming 在此处寻找最新镜像版本 # mysql与es可以使用自己搭建的服务,调整对应配置即可 @@ -137,7 +137,7 @@ version: "2" services: # *不要调整knowstreaming-manager服务名称，ui中会用到 knowstreaming-manager: - image: knowstreaming/knowstreaming-manager:0.2.0-test + image: knowstreaming/knowstreaming-manager:latest container_name: knowstreaming-manager privileged: true restart: always @@ -170,7 +170,7 @@ services: # volumes: # - /ks/manage/log:/logs knowstreaming-ui: - image: knowstreaming/knowstreaming-ui:0.2.0-test1 + image: knowstreaming/knowstreaming-ui:latest container_name: knowstreaming-ui restart: always ports: @@ -204,7 +204,7 @@ services: # es初始化服务，与manager使用同一镜像 # 首次启动es需初始化模版和索引,后续会自动创建 knowstreaming-init: - image: knowstreaming/knowstreaming-manager:0.2.0-test + image: knowstreaming/knowstreaming-manager:latest container_name: knowstreaming-init depends_on: - elasticsearch-single @@ -217,7 +217,7 @@ services: SERVER_ES_ADDRESS: elasticsearch-single:9200 knowstreaming-mysql: - image: knowstreaming/knowstreaming-mysql:0.2.0-test + image: knowstreaming/knowstreaming-mysql:latest container_name: knowstreaming-mysql restart: always environment: From 9a1e89564e98c28f4d5186977ab57933f633d47e Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:44:49 +0800 Subject: [PATCH 21/23] =?UTF-8?q?Update=20=E5=8D=95=E6=9C=BA=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=89=8B=E5=86=8C.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index 713b297f..9e23a2af 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -74,7 +74,7 @@ sh deploy_KnowStreaming-offline.sh ```bash # 相关镜像在Docker Hub都可以下载 # 快速安装(NAMESPACE需要更改为已存在的，安装启动需要几分钟初始化请稍等~) -helm install -n [NAMESPACE] [NAME] http://download.knowstreaming.com/charts/knowstreaming-manager-0.1.3.tgz +helm install -n [NAMESPACE] [NAME] http://download.knowstreaming.com/charts/knowstreaming-manager-0.1.5.tgz # 获取KnowStreaming前端ui的service. 默认nodeport方式. # (http://nodeIP:nodeport，默认用户名密码：admin/admin2022_) From 161d4c45626cfe2f45a73b06d1f0fc13830ec69e Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:46:27 +0800 Subject: [PATCH 22/23] =?UTF-8?q?Update=20=E5=8D=95=E6=9C=BA=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=89=8B=E5=86=8C.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/install_guide/单机部署手册.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install_guide/单机部署手册.md b/docs/install_guide/单机部署手册.md index 9e23a2af..c42e6318 100644 --- a/docs/install_guide/单机部署手册.md +++ b/docs/install_guide/单机部署手册.md @@ -78,7 +78,7 @@ helm install -n [NAMESPACE] [NAME] http://download.knowstreaming.com/charts/know # 获取KnowStreaming前端ui的service. 默认nodeport方式. # (http://nodeIP:nodeport，默认用户名密码：admin/admin2022_) -# `v3.0.0-beta.2`版本开始，默认账号密码为`admin` / `admin`； +# `v3.0.0-beta.2`版本开始（helm chart包版本0.1.4开始），默认账号密码为`admin` / `admin`； # 添加仓库 helm repo add knowstreaming http://download.knowstreaming.com/charts @@ -98,7 +98,7 @@ helm pull knowstreaming/knowstreaming-manager **安装命令** ```bash -# `v3.0.0-beta.3`版本开始(docker镜像为0.3.0版本开始)，默认账号密码为`admin` / `admin`； +# `v3.0.0-beta.2`版本开始(docker镜像为0.2.0版本开始)，默认账号密码为`admin` / `admin`； # https://hub.docker.com/u/knowstreaming 在此处寻找最新镜像版本 # mysql与es可以使用自己搭建的服务,调整对应配置即可 From 3983d73695df135bbc583eb3e0c89187c48af93d Mon Sep 17 00:00:00 2001 From: wangdongfang-aden <49342405+wangdongfang-aden@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:47:40 +0800 Subject: [PATCH 23/23] Update Chart.yaml --- km-dist/helm/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/km-dist/helm/Chart.yaml b/km-dist/helm/Chart.yaml index 4b764af4..b5f5432c 100644 --- a/km-dist/helm/Chart.yaml +++ b/km-dist/helm/Chart.yaml @@ -4,13 +4,13 @@ description: knowstreaming-manager Helm chart type: application -version: 0.1.4 +version: 0.1.5 maintainers: - email: didicloud@didiglobal.com name: didicloud -appVersion: "3.0.0-beta.2" +appVersion: "3.0.0-beta.3" dependencies: - name: knowstreaming-web